1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import os 6 import unittest 7 8 from telemetry import benchmark 9 from telemetry import story 10 from telemetry.internal.results import base_test_results_unittest 11 from telemetry.internal.results import chart_json_output_formatter 12 from telemetry.internal.results import json_output_formatter 13 from telemetry.internal.results import page_test_results 14 from telemetry import page as page_module 15 from telemetry.testing import stream 16 from telemetry.value import failure 17 from telemetry.value import histogram 18 from telemetry.value import improvement_direction 19 from telemetry.value import scalar 20 from telemetry.value import skip 21 from telemetry.value import trace 22 from tracing.trace_data import trace_data 23 24 25 class PageTestResultsTest(base_test_results_unittest.BaseTestResultsUnittest): 26 def setUp(self): 27 story_set = story.StorySet(base_dir=os.path.dirname(__file__)) 28 story_set.AddStory(page_module.Page("http://www.bar.com/", story_set, story_set.base_dir)) 29 story_set.AddStory(page_module.Page("http://www.baz.com/", story_set, story_set.base_dir)) 30 story_set.AddStory(page_module.Page("http://www.foo.com/", story_set, story_set.base_dir)) 31 self.story_set = story_set 32 33 @property 34 def pages(self): 35 return self.story_set.stories 36 37 def testFailures(self): 38 results = page_test_results.PageTestResults() 39 results.WillRunPage(self.pages[0]) 40 results.AddValue( 41 failure.FailureValue(self.pages[0], self.CreateException())) 42 results.DidRunPage(self.pages[0]) 43 44 results.WillRunPage(self.pages[1]) 45 results.DidRunPage(self.pages[1]) 46 47 self.assertEqual(set([self.pages[0]]), results.pages_that_failed) 48 self.assertEqual(set([self.pages[1]]), results.pages_that_succeeded) 49 50 self.assertEqual(2, len(results.all_page_runs)) 51 self.assertTrue(results.all_page_runs[0].failed) 52 self.assertTrue(results.all_page_runs[1].ok) 53 54 def testSkips(self): 55 results = page_test_results.PageTestResults() 56 results.WillRunPage(self.pages[0]) 57 results.AddValue(skip.SkipValue(self.pages[0], 'testing reason')) 58 results.DidRunPage(self.pages[0]) 59 60 results.WillRunPage(self.pages[1]) 61 results.DidRunPage(self.pages[1]) 62 63 self.assertTrue(results.all_page_runs[0].skipped) 64 self.assertEqual(self.pages[0], results.all_page_runs[0].story) 65 self.assertEqual(set([self.pages[0], self.pages[1]]), 66 results.pages_that_succeeded) 67 68 self.assertEqual(2, len(results.all_page_runs)) 69 self.assertTrue(results.all_page_runs[0].skipped) 70 self.assertTrue(results.all_page_runs[1].ok) 71 72 def testBasic(self): 73 results = page_test_results.PageTestResults() 74 results.WillRunPage(self.pages[0]) 75 results.AddValue(scalar.ScalarValue( 76 self.pages[0], 'a', 'seconds', 3, 77 improvement_direction=improvement_direction.UP)) 78 results.DidRunPage(self.pages[0]) 79 80 results.WillRunPage(self.pages[1]) 81 results.AddValue(scalar.ScalarValue( 82 self.pages[1], 'a', 'seconds', 3, 83 improvement_direction=improvement_direction.UP)) 84 results.DidRunPage(self.pages[1]) 85 86 results.PrintSummary() 87 88 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') 89 self.assertEquals(1, len(values)) 90 v = values[0] 91 self.assertEquals(v.name, 'a') 92 self.assertEquals(v.page, self.pages[0]) 93 94 values = results.FindAllPageSpecificValuesNamed('a') 95 assert len(values) == 2 96 97 def testAddValueWithStoryGroupingKeys(self): 98 results = page_test_results.PageTestResults() 99 self.pages[0].grouping_keys['foo'] = 'bar' 100 self.pages[0].grouping_keys['answer'] = '42' 101 results.WillRunPage(self.pages[0]) 102 results.AddValue(scalar.ScalarValue( 103 self.pages[0], 'a', 'seconds', 3, 104 improvement_direction=improvement_direction.UP)) 105 results.DidRunPage(self.pages[0]) 106 107 results.PrintSummary() 108 109 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') 110 v = values[0] 111 self.assertEquals(v.grouping_keys['foo'], 'bar') 112 self.assertEquals(v.grouping_keys['answer'], '42') 113 self.assertEquals(v.tir_label, '42_bar') 114 115 def testAddValueWithStoryGroupingKeysAndMatchingTirLabel(self): 116 results = page_test_results.PageTestResults() 117 self.pages[0].grouping_keys['foo'] = 'bar' 118 self.pages[0].grouping_keys['answer'] = '42' 119 results.WillRunPage(self.pages[0]) 120 results.AddValue(scalar.ScalarValue( 121 self.pages[0], 'a', 'seconds', 3, 122 improvement_direction=improvement_direction.UP, 123 tir_label='42_bar')) 124 results.DidRunPage(self.pages[0]) 125 126 results.PrintSummary() 127 128 values = results.FindPageSpecificValuesForPage(self.pages[0], 'a') 129 v = values[0] 130 self.assertEquals(v.grouping_keys['foo'], 'bar') 131 self.assertEquals(v.grouping_keys['answer'], '42') 132 self.assertEquals(v.tir_label, '42_bar') 133 134 def testAddValueWithStoryGroupingKeysAndMismatchingTirLabel(self): 135 results = page_test_results.PageTestResults() 136 self.pages[0].grouping_keys['foo'] = 'bar' 137 self.pages[0].grouping_keys['answer'] = '42' 138 results.WillRunPage(self.pages[0]) 139 with self.assertRaises(AssertionError): 140 results.AddValue(scalar.ScalarValue( 141 self.pages[0], 'a', 'seconds', 3, 142 improvement_direction=improvement_direction.UP, 143 tir_label='another_label')) 144 145 def testAddValueWithDuplicateStoryGroupingKeyFails(self): 146 results = page_test_results.PageTestResults() 147 self.pages[0].grouping_keys['foo'] = 'bar' 148 results.WillRunPage(self.pages[0]) 149 with self.assertRaises(AssertionError): 150 results.AddValue(scalar.ScalarValue( 151 self.pages[0], 'a', 'seconds', 3, 152 improvement_direction=improvement_direction.UP, 153 grouping_keys={'foo': 'bar'})) 154 155 def testUrlIsInvalidValue(self): 156 results = page_test_results.PageTestResults() 157 results.WillRunPage(self.pages[0]) 158 self.assertRaises( 159 AssertionError, 160 lambda: results.AddValue(scalar.ScalarValue( 161 self.pages[0], 'url', 'string', 'foo', 162 improvement_direction=improvement_direction.UP))) 163 164 def testAddSummaryValueWithPageSpecified(self): 165 results = page_test_results.PageTestResults() 166 results.WillRunPage(self.pages[0]) 167 self.assertRaises( 168 AssertionError, 169 lambda: results.AddSummaryValue(scalar.ScalarValue( 170 self.pages[0], 'a', 'units', 3, 171 improvement_direction=improvement_direction.UP))) 172 173 def testUnitChange(self): 174 results = page_test_results.PageTestResults() 175 results.WillRunPage(self.pages[0]) 176 results.AddValue(scalar.ScalarValue( 177 self.pages[0], 'a', 'seconds', 3, 178 improvement_direction=improvement_direction.UP)) 179 results.DidRunPage(self.pages[0]) 180 181 results.WillRunPage(self.pages[1]) 182 self.assertRaises( 183 AssertionError, 184 lambda: results.AddValue(scalar.ScalarValue( 185 self.pages[1], 'a', 'foobgrobbers', 3, 186 improvement_direction=improvement_direction.UP))) 187 188 def testTypeChange(self): 189 results = page_test_results.PageTestResults() 190 results.WillRunPage(self.pages[0]) 191 results.AddValue(scalar.ScalarValue( 192 self.pages[0], 'a', 'seconds', 3, 193 improvement_direction=improvement_direction.UP)) 194 results.DidRunPage(self.pages[0]) 195 196 results.WillRunPage(self.pages[1]) 197 self.assertRaises( 198 AssertionError, 199 lambda: results.AddValue(histogram.HistogramValue( 200 self.pages[1], 'a', 'seconds', 201 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', 202 improvement_direction=improvement_direction.UP))) 203 204 def testGetPagesThatSucceededAllPagesFail(self): 205 results = page_test_results.PageTestResults() 206 results.WillRunPage(self.pages[0]) 207 results.AddValue(scalar.ScalarValue( 208 self.pages[0], 'a', 'seconds', 3, 209 improvement_direction=improvement_direction.UP)) 210 results.AddValue(failure.FailureValue.FromMessage(self.pages[0], 'message')) 211 results.DidRunPage(self.pages[0]) 212 213 results.WillRunPage(self.pages[1]) 214 results.AddValue(scalar.ScalarValue( 215 self.pages[1], 'a', 'seconds', 7, 216 improvement_direction=improvement_direction.UP)) 217 results.AddValue(failure.FailureValue.FromMessage(self.pages[1], 'message')) 218 results.DidRunPage(self.pages[1]) 219 220 results.PrintSummary() 221 self.assertEquals(0, len(results.pages_that_succeeded)) 222 223 def testGetSuccessfulPageValuesMergedNoFailures(self): 224 results = page_test_results.PageTestResults() 225 results.WillRunPage(self.pages[0]) 226 results.AddValue(scalar.ScalarValue( 227 self.pages[0], 'a', 'seconds', 3, 228 improvement_direction=improvement_direction.UP)) 229 self.assertEquals(1, len(results.all_page_specific_values)) 230 results.DidRunPage(self.pages[0]) 231 232 def testGetAllValuesForSuccessfulPages(self): 233 results = page_test_results.PageTestResults() 234 results.WillRunPage(self.pages[0]) 235 value1 = scalar.ScalarValue( 236 self.pages[0], 'a', 'seconds', 3, 237 improvement_direction=improvement_direction.UP) 238 results.AddValue(value1) 239 results.DidRunPage(self.pages[0]) 240 241 results.WillRunPage(self.pages[1]) 242 value2 = scalar.ScalarValue( 243 self.pages[1], 'a', 'seconds', 3, 244 improvement_direction=improvement_direction.UP) 245 results.AddValue(value2) 246 results.DidRunPage(self.pages[1]) 247 248 results.WillRunPage(self.pages[2]) 249 value3 = scalar.ScalarValue( 250 self.pages[2], 'a', 'seconds', 3, 251 improvement_direction=improvement_direction.UP) 252 results.AddValue(value3) 253 results.DidRunPage(self.pages[2]) 254 255 self.assertEquals( 256 [value1, value2, value3], results.all_page_specific_values) 257 258 def testGetAllValuesForSuccessfulPagesOnePageFails(self): 259 results = page_test_results.PageTestResults() 260 results.WillRunPage(self.pages[0]) 261 value1 = scalar.ScalarValue( 262 self.pages[0], 'a', 'seconds', 3, 263 improvement_direction=improvement_direction.UP) 264 results.AddValue(value1) 265 results.DidRunPage(self.pages[0]) 266 267 results.WillRunPage(self.pages[1]) 268 value2 = failure.FailureValue.FromMessage(self.pages[1], 'Failure') 269 results.AddValue(value2) 270 results.DidRunPage(self.pages[1]) 271 272 results.WillRunPage(self.pages[2]) 273 value3 = scalar.ScalarValue( 274 self.pages[2], 'a', 'seconds', 3, 275 improvement_direction=improvement_direction.UP) 276 results.AddValue(value3) 277 results.DidRunPage(self.pages[2]) 278 279 self.assertEquals( 280 [value1, value2, value3], results.all_page_specific_values) 281 282 def testFindValues(self): 283 results = page_test_results.PageTestResults() 284 results.WillRunPage(self.pages[0]) 285 v0 = scalar.ScalarValue( 286 self.pages[0], 'a', 'seconds', 3, 287 improvement_direction=improvement_direction.UP) 288 results.AddValue(v0) 289 v1 = scalar.ScalarValue( 290 self.pages[0], 'a', 'seconds', 4, 291 improvement_direction=improvement_direction.UP) 292 results.AddValue(v1) 293 results.DidRunPage(self.pages[1]) 294 295 values = results.FindValues(lambda v: v.value == 3) 296 self.assertEquals([v0], values) 297 298 def testValueWithTIRLabel(self): 299 results = page_test_results.PageTestResults() 300 results.WillRunPage(self.pages[0]) 301 v0 = scalar.ScalarValue( 302 self.pages[0], 'a', 'seconds', 3, tir_label='foo', 303 improvement_direction=improvement_direction.UP) 304 results.AddValue(v0) 305 v1 = scalar.ScalarValue( 306 self.pages[0], 'a', 'seconds', 3, tir_label='bar', 307 improvement_direction=improvement_direction.UP) 308 results.AddValue(v1) 309 results.DidRunPage(self.pages[0]) 310 311 values = results.FindAllPageSpecificValuesFromIRNamed('foo', 'a') 312 self.assertEquals([v0], values) 313 314 def testTraceValue(self): 315 results = page_test_results.PageTestResults() 316 results.WillRunPage(self.pages[0]) 317 results.AddValue(trace.TraceValue( 318 None, trace_data.CreateTraceDataFromRawData([[{'test': 1}]]))) 319 results.DidRunPage(self.pages[0]) 320 321 results.WillRunPage(self.pages[1]) 322 results.AddValue(trace.TraceValue( 323 None, trace_data.CreateTraceDataFromRawData([[{'test': 2}]]))) 324 results.DidRunPage(self.pages[1]) 325 326 results.PrintSummary() 327 328 values = results.FindAllTraceValues() 329 self.assertEquals(2, len(values)) 330 331 def testCleanUpCleansUpTraceValues(self): 332 results = page_test_results.PageTestResults() 333 v0 = trace.TraceValue( 334 None, trace_data.CreateTraceDataFromRawData([{'test': 1}])) 335 v1 = trace.TraceValue( 336 None, trace_data.CreateTraceDataFromRawData([{'test': 2}])) 337 338 results.WillRunPage(self.pages[0]) 339 results.AddValue(v0) 340 results.DidRunPage(self.pages[0]) 341 342 results.WillRunPage(self.pages[1]) 343 results.AddValue(v1) 344 results.DidRunPage(self.pages[1]) 345 346 results.CleanUp() 347 self.assertTrue(v0.cleaned_up) 348 self.assertTrue(v1.cleaned_up) 349 350 def testNoTracesLeftAfterCleanUp(self): 351 results = page_test_results.PageTestResults() 352 v0 = trace.TraceValue(None, 353 trace_data.CreateTraceDataFromRawData([{'test': 1}])) 354 v1 = trace.TraceValue(None, 355 trace_data.CreateTraceDataFromRawData([{'test': 2}])) 356 357 results.WillRunPage(self.pages[0]) 358 results.AddValue(v0) 359 results.DidRunPage(self.pages[0]) 360 361 results.WillRunPage(self.pages[1]) 362 results.AddValue(v1) 363 results.DidRunPage(self.pages[1]) 364 365 results.CleanUp() 366 self.assertFalse(results.FindAllTraceValues()) 367 368 def testPrintSummaryDisabledResults(self): 369 output_stream = stream.TestOutputStream() 370 output_formatters = [] 371 benchmark_metadata = benchmark.BenchmarkMetadata( 372 'benchmark_name', 'benchmark_description') 373 output_formatters.append( 374 chart_json_output_formatter.ChartJsonOutputFormatter( 375 output_stream, benchmark_metadata)) 376 output_formatters.append(json_output_formatter.JsonOutputFormatter( 377 output_stream, benchmark_metadata)) 378 results = page_test_results.PageTestResults( 379 output_formatters=output_formatters, benchmark_enabled=False) 380 results.PrintSummary() 381 self.assertEquals(output_stream.output_data, 382 "{\n \"enabled\": false,\n \"benchmark_name\": \"benchmark_name\"\n}\n") 383 384 385 class PageTestResultsFilterTest(unittest.TestCase): 386 def setUp(self): 387 story_set = story.StorySet(base_dir=os.path.dirname(__file__)) 388 story_set.AddStory( 389 page_module.Page('http://www.foo.com/', story_set, story_set.base_dir)) 390 story_set.AddStory( 391 page_module.Page('http://www.bar.com/', story_set, story_set.base_dir)) 392 self.story_set = story_set 393 394 @property 395 def pages(self): 396 return self.story_set.stories 397 398 def testFilterValue(self): 399 def AcceptValueNamed_a(value, _): 400 return value.name == 'a' 401 results = page_test_results.PageTestResults( 402 value_can_be_added_predicate=AcceptValueNamed_a) 403 results.WillRunPage(self.pages[0]) 404 results.AddValue(scalar.ScalarValue( 405 self.pages[0], 'a', 'seconds', 3, 406 improvement_direction=improvement_direction.UP)) 407 results.AddValue(scalar.ScalarValue( 408 self.pages[0], 'b', 'seconds', 3, 409 improvement_direction=improvement_direction.UP)) 410 results.DidRunPage(self.pages[0]) 411 412 results.WillRunPage(self.pages[1]) 413 results.AddValue(scalar.ScalarValue( 414 self.pages[1], 'a', 'seconds', 3, 415 improvement_direction=improvement_direction.UP)) 416 results.AddValue(scalar.ScalarValue( 417 self.pages[1], 'd', 'seconds', 3, 418 improvement_direction=improvement_direction.UP)) 419 results.DidRunPage(self.pages[1]) 420 results.PrintSummary() 421 self.assertEquals( 422 [('a', 'http://www.foo.com/'), ('a', 'http://www.bar.com/')], 423 [(v.name, v.page.url) for v in results.all_page_specific_values]) 424 425 def testFilterIsFirstResult(self): 426 def AcceptSecondValues(_, is_first_result): 427 return not is_first_result 428 results = page_test_results.PageTestResults( 429 value_can_be_added_predicate=AcceptSecondValues) 430 431 # First results (filtered out) 432 results.WillRunPage(self.pages[0]) 433 results.AddValue(scalar.ScalarValue( 434 self.pages[0], 'a', 'seconds', 7, 435 improvement_direction=improvement_direction.UP)) 436 results.AddValue(scalar.ScalarValue( 437 self.pages[0], 'b', 'seconds', 8, 438 improvement_direction=improvement_direction.UP)) 439 results.DidRunPage(self.pages[0]) 440 results.WillRunPage(self.pages[1]) 441 results.AddValue(scalar.ScalarValue( 442 self.pages[1], 'a', 'seconds', 5, 443 improvement_direction=improvement_direction.UP)) 444 results.AddValue(scalar.ScalarValue( 445 self.pages[1], 'd', 'seconds', 6, 446 improvement_direction=improvement_direction.UP)) 447 results.DidRunPage(self.pages[1]) 448 449 # Second results 450 results.WillRunPage(self.pages[0]) 451 results.AddValue(scalar.ScalarValue( 452 self.pages[0], 'a', 'seconds', 3, 453 improvement_direction=improvement_direction.UP)) 454 results.AddValue(scalar.ScalarValue( 455 self.pages[0], 'b', 'seconds', 4, 456 improvement_direction=improvement_direction.UP)) 457 results.DidRunPage(self.pages[0]) 458 results.WillRunPage(self.pages[1]) 459 results.AddValue(scalar.ScalarValue( 460 self.pages[1], 'a', 'seconds', 1, 461 improvement_direction=improvement_direction.UP)) 462 results.AddValue(scalar.ScalarValue( 463 self.pages[1], 'd', 'seconds', 2, 464 improvement_direction=improvement_direction.UP)) 465 results.DidRunPage(self.pages[1]) 466 results.PrintSummary() 467 expected_values = [ 468 ('a', 'http://www.foo.com/', 3), 469 ('b', 'http://www.foo.com/', 4), 470 ('a', 'http://www.bar.com/', 1), 471 ('d', 'http://www.bar.com/', 2)] 472 actual_values = [(v.name, v.page.url, v.value) 473 for v in results.all_page_specific_values] 474 self.assertEquals(expected_values, actual_values) 475 476 def testFailureValueCannotBeFiltered(self): 477 def AcceptValueNamed_a(value, _): 478 return value.name == 'a' 479 results = page_test_results.PageTestResults( 480 value_can_be_added_predicate=AcceptValueNamed_a) 481 results.WillRunPage(self.pages[0]) 482 results.AddValue(scalar.ScalarValue( 483 self.pages[0], 'b', 'seconds', 8, 484 improvement_direction=improvement_direction.UP)) 485 failure_value = failure.FailureValue.FromMessage(self.pages[0], 'failure') 486 results.AddValue(failure_value) 487 results.DidRunPage(self.pages[0]) 488 results.PrintSummary() 489 490 # Although predicate says only accept values named 'a', the failure value is 491 # added anyway. 492 self.assertEquals(len(results.all_page_specific_values), 1) 493 self.assertIn(failure_value, results.all_page_specific_values) 494 495 def testSkipValueCannotBeFiltered(self): 496 def AcceptValueNamed_a(value, _): 497 return value.name == 'a' 498 results = page_test_results.PageTestResults( 499 value_can_be_added_predicate=AcceptValueNamed_a) 500 results.WillRunPage(self.pages[0]) 501 skip_value = skip.SkipValue(self.pages[0], 'skip for testing') 502 results.AddValue(scalar.ScalarValue( 503 self.pages[0], 'b', 'seconds', 8, 504 improvement_direction=improvement_direction.UP)) 505 results.AddValue(skip_value) 506 results.DidRunPage(self.pages[0]) 507 results.PrintSummary() 508 509 # Although predicate says only accept value with named 'a', skip value is 510 # added anyway. 511 self.assertEquals(len(results.all_page_specific_values), 1) 512 self.assertIn(skip_value, results.all_page_specific_values) 513
