1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 # 5 # This file lists the pyauto tests that run as a part of the functional test 6 # suite. 7 # 8 # Tests can be enabled on a per-platform basis. Tests not listed here will 9 # not be run. 10 # 11 # Within each platform, tests are specified in up to 3 groups: (1) enabled 12 # tests; (2) tests that are permanently disabled because they do not apply 13 # to the given platform and hence should not be run; and (3) disabled tests 14 # that need to be investigated/fixed. Tests are listed alphabetically within 15 # each group. 16 # 17 # Test names can be specified in any of the following ways: 18 # 1. as a module, in which case all tests in that module will be run 19 # example: 'test_basic' 20 # 2. or as a test case, in which case all tests in that test case will be run 21 # example: 'test_basic.SimpleTest' 22 # 3. or as an individual test 23 # example: 'test_basic.SimpleTest.testCanOpenGoogle' 24 # 25 # Tests beginning with '-' will be excluded. This can be used to enforce 26 # exclusions for a particular platform. 27 # Names beginning with '@' will be treated as suite names and will be 28 # recursively expanded. 29 30 { 31 # This suite gets run on 'Google Chrome' builds. 32 'FULL': { 33 'all': [ 34 'about_plugins_ui.AboutPluginsUITest', 35 'about_plugins_ui.ChromeAboutPluginsUITest', 36 'apptest', 37 'autofill', 38 'codesign', 39 'crash_reporter', 40 'execute_javascript', 41 'extensions', 42 'fullscreen_mouselock', 43 'gtalk.test_basic', 44 'infobars', 45 'media.media_basic_playback', 46 'multiprofile', 47 'nacl_sdk', 48 'ntp', 49 'omnibox', 50 'passwords', 51 'prefs', 52 'prefs_ui', 53 'pyauto_webdriver', 54 'search_engines', 55 'special_tabs', 56 'test_pyauto', 57 58 # =========================== 59 # Permanently-disabled tests. 60 # =========================== 61 62 # ================================================== 63 # Disabled tests that need to be investigated/fixed. 64 # ================================================== 65 # crbug.com/145006 66 '-about_plugins_ui.ChromeAboutPluginsUITest.testEnableAndDisableFlashPlugin', 67 # crbug.com/100365 68 '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', 69 # crbug.com/171828 70 '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs', 71 # The source is behind. Waiting for dev to automate the update. 72 # crbug.com/109160 73 '-execute_javascript.ExecuteJavascriptTest.testExecuteJavascriptInExtension', 74 # crbug.com/123396 75 '-fullscreen_mouselock.FullscreenMouselockTest.testPatternsForFSAndML', 76 # crbug.com/132665 77 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavBackToPrevPage', 78 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavToNewPage', 79 '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavBackToPrevPage', 80 '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavToNewPage', 81 # crbug.com/140460 82 '-fullscreen_mouselock.FullscreenMouselockTest.testNoTabFSExitWhenJSExitMouseLock', 83 # crosbug.com/136875 84 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSDoesNotExitForAnchorLinks', 85 # crbug.com/179263 86 '-gtalk.test_basic.BasicTest.testCurrentVersion', 87 '-gtalk.test_basic.BasicTest.testRCVersion', 88 # crbug.com/165796 89 '-infobars.OneClickInfobarTest', 90 # crbug.com/131874 91 '-infobars.OneClickInfobarTest.testNoOneClickInfobarAfterCancel', 92 # crbug.com/133315 93 '-infobars.OneClickInfobarTest.testDisplayOneClickInfobarAfterDismiss', 94 # Mysteriously broken? 95 # crbug.com/138857 96 '-multiprofile.MultiprofileTest.testMakeSearchEngineDefaultInMultiprofile', 97 # crbug.com/179268 98 '-ntp.NTPTest.testCannotUninstallWebStore', 99 '-ntp.NTPTest.testGetAppsInNewProfile', 100 '-ntp.NTPTest.testGetAppsWhenInstallApp', 101 '-ntp.NTPTest.testGetAppsWhenInstallNonApps', 102 '-ntp.NTPTest.testUninstallApp', 103 # crbug.com/143308 104 '-omnibox.OmniboxLiveTest.testGoogleSearch', 105 # crbug.com/71715 106 '-omnibox.OmniboxTest.testHistoryResult', 107 # crbug.com/123019 108 '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', 109 # crbug.com/137041 110 '-omnibox.OmniboxTest.testDifferentTypesOfResults', 111 # crbug.com/162341 112 '-prefs.PrefsTest.testAllowSelectedGeoTracking', 113 '-prefs.PrefsTest.testDismissedInfobarSavesNoEntry', 114 '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', 115 '-prefs.PrefsTest.testGeolocationPref', 116 # crbug.com/85600 117 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', 118 # crbug.com/157271 119 '-prefs_ui.BasicSettingsUITest.testCancelStartupURLSetting', 120 '-prefs_ui.BasicSettingsUITest.testSetStartupPages', 121 '-prefs_ui.BasicSettingsUITest.testUseCurrentPagesForStartup', 122 # crbug.com/162341 123 '-prefs_ui.PrefsUITest.testBehaviorValueCorrectlyDisplayed', 124 # crbug.com/112051 125 '-prefs_ui.PrefsUITest.testChangeExceptionBehaviorUI', 126 # crbug.com/157271 127 '-prefs_ui.PrefsUITest.testDeleteExceptionUI', 128 # crbug.com/151973 129 '-prefs_ui.PrefsUITest.testInitialLineEntryInIncognitoUI', 130 # crbug.com/162341 131 '-prefs_ui.PrefsUITest.testLocationSettingOptionsUI', 132 '-prefs_ui.PrefsUITest.testNoInitialLineEntryInUI', 133 # crbug.com/132285 134 '-special_tabs.SpecialTabsTest.testSpecialURLTabs', 135 # crbug.com/168081 136 '-ntp.NTPTest.testCloseOneTab', 137 ], 138 139 'win': [ 140 'gpu', 141 # =========================== 142 # Permanently-disabled tests. 143 # =========================== 144 145 # ================================================== 146 # Disabled tests that need to be investigated/fixed. 147 # ================================================== 148 # crbug.com/105948 149 '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', 150 # crbug.com/111289 151 '-extensions.ExtensionsTest.testAllowAccessFileURLs', 152 # crbug.com/113090 153 '-extensions.ExtensionsTest.testAllowIncognitoExtension', 154 # crbug.com/171490 155 '-passwords.PasswordTest.testPasswdInfoNotStoredWhenAutocompleteOff', 156 # crbug.com/117569 157 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', 158 # crbug.com/98526 159 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', 160 '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', 161 ], 162 163 'mac': [ 164 # =========================== 165 # Permanently-disabled tests. 166 # =========================== 167 168 # ================================================== 169 # Disabled tests that need to be investigated/fixed. 170 # ================================================== 171 # Keychain popups make autofill/password tests difficult: crbug.com/49378 172 '-prefs_ui.PrefsUITest.testSetPasswordAndDelete', 173 # codesign tests should run *after* signing. crbug.com/50481 174 '-codesign', 175 # crbug.com/124922 176 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenBrowserLoseFocus', 177 # crbug.com/125989 178 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenAlertDialogShow', 179 # Fails on chrome-mac-10_7-qa only: crbug.com/124886 180 '-fullscreen_mouselock.FullscreenMouselockTest.testPrefsForFullscreenExit', 181 '-fullscreen_mouselock.FullscreenMouselockTest.testNoMouseLockWhenCancelFS', 182 # crbug.com/121484 183 '-multiprofile.MultiprofileTest.test20NewProfiles', 184 '-ntp.NTPTest.testLaunchAppNewWindow', # crbug.com/79812 185 # crbug.com/70437 186 '-omnibox.OmniboxTest.testHistoryResult', 187 # crbug.com/91617 188 '-omnibox.OmniboxTest.testContentHistory', 189 # Keychain popups make autofill/password tests difficult: crbug.com/49378 190 '-passwords', 191 # crbug.com/69619 192 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', 193 # crbug.com/98526 194 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', 195 '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', 196 ], 197 198 'linux': [ 199 'test_clean_exit', 200 201 # =========================== 202 # Permanently-disabled tests. 203 # =========================== 204 # System password manager obstructs password automation. 205 '-passwords', 206 207 # ================================================== 208 # Disabled tests that need to be investigated/fixed. 209 # ================================================== 210 # crbug.com/111289 211 '-extensions.ExtensionsTest.testAllowAccessFileURLs', 212 # crbug.com/91033 213 '-omnibox.OmniboxTest.testOmniboxSearchHistory', 214 ], 215 216 'chromeos': [ 217 'chromeos_basic', 218 'chromeos_browser', 219 'chromeos_crosh', 220 'chromeos_file_browser', 221 'chromeos_offline', 222 'chromeos_power', 223 'chromeos_prefs', 224 'chromeos_security', 225 'chromeos_time', 226 'chromeos_wifi_sanity', 227 'doc_viewing', 228 'secure_shell', 229 'youtube', 230 231 # =========================== 232 # Permanently-disabled tests. 233 # =========================== 234 # No codesign verification on ChromeOS. 235 '-codesign', 236 # Sync is already signed in with the login account. 237 # So one-click infobar tests do tno apply to chromeos. 238 '-infobars.OneClickInfobarTest', 239 # Multi-profile doesn't apply to chromeos yet. 240 '-multiprofile', 241 '-ntp.NTPTest.testDifferentProfileNotAppearInMostVisited', 242 # No NaCl support on ChromeOS. 243 '-nacl_sdk', 244 245 # ================================================== 246 # Disabled tests that need to be investigated/fixed. 247 # ================================================== 248 # crbug.com/132337 249 '-autofill.AutofillTest.testTabOrderForEditAddress', 250 # crosbug.com/19556 251 '-extensions.ExtensionsTest.testAllowAccessFileURLs', 252 '-extensions.ExtensionsTest.testAllowIncognitoExtension', 253 '-extensions.ExtensionsTest.testDisableEnableExtension', 254 # crbug.com/134593 255 '-gtalk.test_basic.BasicTest.testCurrentVersion', 256 '-gtalk.test_basic.BasicTest.testRCVersion', 257 # crosbug.com/24496 258 '-infobars.InfobarTest.testPluginCrashForMultiTabs', 259 # crbug.com/109035 260 '-infobars.InfobarTest.testPluginCrashInfobar', 261 # crosbug.com/14256 262 '-ntp.NTPTest.testLaunchAppFullScreen', 263 # Content history broken in omnibox. crosbug.com/14416 264 '-omnibox.OmniboxTest.testContentHistory', 265 # crbug.com/91033 266 '-omnibox.OmniboxTest.testOmniboxSearchHistory', 267 # crosbug.com/19760 268 '-passwords.PasswordTest.testClearFetchedCredForNewUserName', 269 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', 270 # onunload popups get created in the same window on chromeos 271 # Session restore not working with PyAuto. crosbug.com/12648 272 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', 273 '-prefs.PrefsTest.testSessionRestoreURLs', 274 '-prefs.PrefsTest.testSessionRestore', 275 # Deal with i18n chars. crosbug.com/12639 276 '-omnibox.OmniboxTest.testCrazyFilenames', 277 # crosbug.com/16977 278 '-chromeos_wifi_sanity.ChromeosWifiSanity.testConnectToHiddenWiFiNonExistent', 279 # crosbug.com/20025 280 '-chromeos_browser.ChromeosBrowserTest.testFullScreen', 281 # Chrome driver does not work in Chrome OS. 282 # crosbug.com/19556 283 '-prefs_ui', 284 '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', 285 '-prefs.PrefsTest.testGeolocationPref', 286 '-pyauto_webdriver', 287 ], 288 }, 289 290 # Performance tests. 291 'PERFORMANCE': { 292 'all': [ 293 'perf', 294 295 # ================================================== 296 # Disabled tests that need to be investigated/fixed. 297 # ================================================== 298 '-perf.HTML5BenchmarkTest', # crbug.com/134476 299 '-perf.LiveWebappLoadTest.testNewTabGmail', # crbug.com/136554 300 '-perf.ScrollTest.testGmailScroll', # crbug.com/136554 301 '-perf.WebGLTest.testWebGLField', # crbug.com/132797 302 303 # =========================== 304 # Permanently-disabled tests. 305 # =========================== 306 # Invoked outside of the pyauto_tests framework. 307 '-perf.BenchmarkPerfTest.testSpaceport', 308 '-perf.PopularSitesScrollTest.test2012Q3', 309 ], 310 'win': [ 311 # ================================================== 312 # Disabled tests that need to be investigated/fixed. 313 # ================================================== 314 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. 315 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 316 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. 317 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. 318 319 # =========================== 320 # Permanently-disabled tests. 321 # =========================== 322 '-perf.MemoryTest', # Designed only for ChromeOS. 323 '-perf.NetflixPerfTest', # Designed only for ChromeOS. 324 ], 325 'mac': [ 326 # ================================================== 327 # Disabled tests that need to be investigated/fixed. 328 # ================================================== 329 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. 330 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 331 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. 332 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. 333 334 # =========================== 335 # Permanently-disabled tests. 336 # =========================== 337 '-perf.MemoryTest', # Designed only for ChromeOS. 338 '-perf.NetflixPerfTest', # Designed only for ChromeOS. 339 ], 340 'linux': [ 341 # ================================================== 342 # Disabled tests that need to be investigated/fixed. 343 # ================================================== 344 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. 345 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 346 '-perf.WebGLTest', # May not render WebGL; need to verify on QA bot. 347 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. 348 349 # =========================== 350 # Permanently-disabled tests. 351 # =========================== 352 '-perf.MemoryTest', # Designed only for ChromeOS. 353 '-perf.NetflixPerfTest', # Designed only for ChromeOS. 354 ], 355 'chromeos': [ 356 # ================================================== 357 # Disabled tests that need to be investigated/fixed. 358 # ================================================== 359 '-perf.NetflixPerfTest', # crosbug.com/32320. 360 '-perf.PageCyclerNetSimTest', # Investigate Web Page Replay on ChromeOS. 361 '-perf.YoutubePerfTest', # crbug.com/233706 362 363 # =========================== 364 # Permanently-disabled tests. 365 # =========================== 366 '-perf.GPUPerfTest', # Designed only for Chrome desktop. 367 ], 368 }, 369 370 'PERFORMANCE_LAB': { 371 'all': [ 372 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', 373 'perf.FlashTest.testFlashGaming', 374 'perf.ScrollTest.testGooglePlusScroll', 375 'perf.TabPerfTest.test20Tabs', 376 'perf.WebGLTest.testWebGLSpaceRocks', 377 ], 378 }, 379 380 'EMPTY': { 381 }, 382 383 'CHROMEOS_CONNECTIVITY': { 384 'chromeos': [ 385 'chromeos_wifi_functional', 386 'chromeos_wifi_compliance', 387 'wifi_downloads', 388 'wifi_notification', 389 ], 390 }, 391 392 # ChromeOS flash tests. 393 'CHROMEOS_FLASH': { 394 'chromeos': [ 395 'flash', 396 ], 397 }, 398 399 # ChromeOS login tests. 400 'CHROMEOS_LOGIN': { 401 'chromeos': [ 402 'chromeos_login', 403 'chromeos_oobe', 404 405 # crosbug.com/32583 406 '-chromeos_login.ChromeosLoginCachedCredentialsUserPod.testCachedCredentialsUserPod', 407 ], 408 }, 409 410 # ChromeOS policy tests. Some of these are shared with Desktop Chrome tests 411 # but require custom setup on ChromeOS. Others are specific to ChromeOS only. 412 'CHROMEOS_POLICY': { 413 'chromeos': [ 414 'chromeos_ephemeral', 415 'chromeos_device_policy', 416 'chromeos_onc', 417 'chromeos_retail_mode', 418 ], 419 }, 420 421 # ChromeOS volume tests. 422 'CHROMEOS_VOLUME': { 423 'chromeos': [ 424 'chromeos_volume', 425 ], 426 }, 427 428 # ChromeOS Accessibility tests. 429 'CHROMEOS_ACCESSIBILITY': { 430 'chromeos': [ 431 'chromeos_accessibility', 432 ], 433 }, 434 435 # PGO performance suite. 436 'PGO': { 437 'chromeos': [ 438 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', 439 'perf.PageCyclerTest', 440 'perf.ScrollTest.testGooglePlusScroll', 441 ], 442 }, 443 444 # Subset of ChromeOS performance tests for seaboard. 445 'CHROMEOS_PERF_SEABOARD': { 446 'chromeos': [ 447 'perf.ScrollTest.testBlankPageScroll', 448 'perf.ScrollTest.testGooglePlusScroll', 449 'perf.ScrollTest.testTextScroll', 450 ], 451 }, 452 453 # HTML5 media performance tests. 454 'AV_PERF': { 455 'linux': [ 456 'media.audio_latency_perf', 457 'media.audio_playback_perf', 458 'media.media_constrained_network_perf', 459 'media.media_scrub_perf', 460 'media.media_seek_perf', 461 'media.media_stat_perf', 462 'media.mixed_audio_latency_perf', 463 ], 464 'win': [ 465 'media.audio_latency_perf', 466 'media.audio_playback_perf', 467 'media.media_constrained_network_perf', 468 'media.media_scrub_perf', 469 'media.media_seek_perf', 470 'media.media_stat_perf', 471 'media.mixed_audio_latency_perf', 472 ], 473 }, 474 475 # WebRTC MediaStream tests. Requires webcam and audio device to be present on 476 # the test machine. 477 'WEBRTC': { 478 'all': [ 479 'media_stream_infobar', 480 'webrtc_brutality_test', 481 'webrtc_call', 482 483 # ================================================== 484 # Disabled tests that need to be investigated/fixed. 485 # ================================================== 486 # crbug.com/248079 487 '-webrtc_brutality_test.WebrtcBrutalityTest.testReloadsAfterGetUserMedia', 488 ], 489 }, 490 491 # WebRTC / AppRTC Integration tests. 492 'WEBRTC_APPRTC': { 493 'all': [ 494 'webrtc_apprtc_call', 495 # ================================================== 496 # Disabled tests that need to be investigated/fixed. 497 # ================================================== 498 # crbug.com/254412 499 '-webrtc_apprtc_call.WebrtcApprtcCallTest.testApprtcTabToTabCall', 500 '-webrtc_apprtc_call.WebrtcApprtcCallTest.testApprtcLoopbackCall', 501 ], 502 }, 503 504 # WebRTC quality tests. The video quality test requires a webcam, an audio 505 # device and a special setup where the webcam plays a barcode-encoded video. 506 # The webrtc_audio_quality test requires additional configuration as described 507 # in the test's class comments. 508 'WEBRTC_QUALITY': { 509 'all': [ 510 'webrtc_video_quality', 511 # Disabled until crbug.com/254437 is resolved. 512 '-webrtc_audio_quality', 513 ], 514 'mac': [ 515 '-webrtc_audio_quality', # Not implemented. 516 ], 517 }, 518 519 # Trace event tests. 520 'TRACING': { 521 'all': [ 522 'tracing.tracing_smoke_test', 523 ], 524 }, 525 526 # Chromoting tests. 527 'CHROMOTING': { 528 'all': [ 529 'chromoting.auth', 530 'chromoting.it2me_basic', 531 'chromoting.me2me_connect', 532 'chromoting.me2me_enable', 533 ], 534 'linux': [ 535 # TODO(yihongg): Me2me test against Linux is not working yet. 536 '-chromoting.me2me_connect', 537 '-chromoting.me2me_enable', 538 ], 539 'chromeos': [ 540 # ChromeOS doesn't yet support the chromoting host. 541 '-chromoting.auth', 542 '-chromoting.it2me_basic', 543 '-chromoting.me2me_connect', 544 '-chromoting.me2me_enable', 545 ], 546 }, 547 548 # Pyauto functional tests running on coverage bots. 549 'CODE_COVERAGE': { 550 'all': [ 551 '@FULL', 552 ], 553 'linux': [ 554 # These tests fail on coverage bots. Disabling for now. 555 '-test_clean_exit', 556 '-about_plugins_ui', 557 '-autofill', 558 '-pyauto_webdriver', 559 '-extensions', 560 '-flash', 561 '-fullscreen_mouselock', 562 '-gtalk.test_basic', 563 '-infobars', 564 '-multiprofile', 565 '-prefs', 566 '-prefs_ui', 567 '-apptest', 568 '-plugins', 569 '-omnibox', 570 '-special_tabs', 571 '-ntp.NTPTest.testLaunchAppFullScreen', 572 '-media.media_basic_playback.MediaBasicPlaybackTest.testBasicPlaybackMatrix', 573 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', 574 '-ntp.NTPTest.testUninstallApp', 575 '-ntp.NTPTest.testLaunchAppWithDefaultSettings', 576 '-ntp.NTPTest.testLaunchAppRegularTab', 577 '-ntp.NTPTest.testLaunchAppPinnedTab', 578 '-ntp.NTPTest.testGetAppsWhenInstallApp', 579 '-ntp.NTPTest.testLaunchAppNewWindow', 580 ], 581 }, 582 } 583
