Home | History | Annotate | Download | only in frame_rate 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include <map>
      6 
      7 #include "base/command_line.h"
      8 #include "base/file_util.h"
      9 #include "base/path_service.h"
     10 #include "base/strings/string_number_conversions.h"
     11 #include "base/strings/utf_string_conversions.h"
     12 #include "base/test/test_timeouts.h"
     13 #include "base/test/trace_event_analyzer.h"
     14 #include "chrome/common/chrome_paths.h"
     15 #include "chrome/common/chrome_switches.h"
     16 #include "chrome/test/automation/automation_proxy.h"
     17 #include "chrome/test/automation/tab_proxy.h"
     18 #include "chrome/test/perf/perf_test.h"
     19 #include "chrome/test/ui/javascript_test_util.h"
     20 #include "chrome/test/ui/ui_perf_test.h"
     21 #include "net/base/net_util.h"
     22 #include "ui/gl/gl_implementation.h"
     23 #include "ui/gl/gl_switches.h"
     24 
     25 #if defined(OS_WIN)
     26 #include "base/win/windows_version.h"
     27 #endif
     28 
     29 namespace {
     30 
     31 enum FrameRateTestFlags {
     32   kUseGpu             = 1 << 0, // Only execute test if --enable-gpu, and verify
     33                                 // that test ran on GPU. This is required for
     34                                 // tests that run on GPU.
     35   kForceGpuComposited = 1 << 1, // Force the test to use the compositor.
     36   kDisableVsync       = 1 << 2, // Do not limit framerate to vertical refresh.
     37                                 // when on GPU, nor to 60hz when not on GPU.
     38   kUseReferenceBuild  = 1 << 3, // Run test using the reference chrome build.
     39   kInternal           = 1 << 4, // Test uses internal test data.
     40   kHasRedirect        = 1 << 5, // Test page contains an HTML redirect.
     41   kIsGpuCanvasTest    = 1 << 6  // Test uses GPU accelerated canvas features.
     42 };
     43 
     44 class FrameRateTest
     45   : public UIPerfTest
     46   , public ::testing::WithParamInterface<int> {
     47  public:
     48   FrameRateTest() {
     49     show_window_ = true;
     50     dom_automation_enabled_ = true;
     51   }
     52 
     53   bool HasFlag(FrameRateTestFlags flag) const {
     54     return (GetParam() & flag) == flag;
     55   }
     56 
     57   bool IsGpuAvailable() const {
     58     return CommandLine::ForCurrentProcess()->HasSwitch("enable-gpu");
     59   }
     60 
     61   std::string GetSuffixForTestFlags() {
     62     std::string suffix;
     63     if (HasFlag(kForceGpuComposited))
     64       suffix += "_comp";
     65     if (HasFlag(kUseGpu))
     66       suffix += "_gpu";
     67     if (HasFlag(kDisableVsync))
     68       suffix += "_novsync";
     69     if (HasFlag(kUseReferenceBuild))
     70       suffix += "_ref";
     71     return suffix;
     72   }
     73 
     74   virtual base::FilePath GetDataPath(const std::string& name) {
     75     // Make sure the test data is checked out.
     76     base::FilePath test_path;
     77     PathService::Get(chrome::DIR_TEST_DATA, &test_path);
     78     test_path = test_path.Append(FILE_PATH_LITERAL("perf"));
     79     test_path = test_path.Append(FILE_PATH_LITERAL("frame_rate"));
     80     if (HasFlag(kInternal)) {
     81       test_path = test_path.Append(FILE_PATH_LITERAL("private"));
     82     } else {
     83       test_path = test_path.Append(FILE_PATH_LITERAL("content"));
     84     }
     85     test_path = test_path.AppendASCII(name);
     86     return test_path;
     87   }
     88 
     89   virtual void SetUp() {
     90     if (HasFlag(kUseReferenceBuild))
     91       UseReferenceBuild();
     92 
     93     // Turn on chrome.Interval to get higher-resolution timestamps on frames.
     94     launch_arguments_.AppendSwitch(switches::kEnableBenchmarking);
     95 
     96     // UI tests boot up render views starting from about:blank. This causes
     97     // the renderer to start up thinking it cannot use the GPU. To work
     98     // around that, and allow the frame rate test to use the GPU, we must
     99     // pass kAllowWebUICompositing.
    100     launch_arguments_.AppendSwitch(switches::kAllowWebUICompositing);
    101 
    102     // Some of the tests may launch http requests through JSON or AJAX
    103     // which causes a security error (cross domain request) when the page
    104     // is loaded from the local file system ( file:// ). The following switch
    105     // fixes that error.
    106     launch_arguments_.AppendSwitch(switches::kAllowFileAccessFromFiles);
    107 
    108     if (!HasFlag(kUseGpu)) {
    109       launch_arguments_.AppendSwitch(switches::kDisableAcceleratedCompositing);
    110       launch_arguments_.AppendSwitch(switches::kDisableExperimentalWebGL);
    111       launch_arguments_.AppendSwitch(switches::kDisableAccelerated2dCanvas);
    112     }
    113 
    114     if (HasFlag(kDisableVsync))
    115       launch_arguments_.AppendSwitch(switches::kDisableGpuVsync);
    116 
    117     UIPerfTest::SetUp();
    118   }
    119 
    120   bool DidRunOnGpu(const std::string& json_events) {
    121     using trace_analyzer::Query;
    122     using trace_analyzer::TraceAnalyzer;
    123 
    124     // Check trace for GPU accleration.
    125     scoped_ptr<TraceAnalyzer> analyzer(TraceAnalyzer::Create(json_events));
    126 
    127     gfx::GLImplementation gl_impl = gfx::kGLImplementationNone;
    128     const trace_analyzer::TraceEvent* gpu_event = analyzer->FindFirstOf(
    129         Query::EventNameIs("SwapBuffers") &&
    130         Query::EventHasNumberArg("GLImpl"));
    131     if (gpu_event)
    132       gl_impl = static_cast<gfx::GLImplementation>(
    133           gpu_event->GetKnownArgAsInt("GLImpl"));
    134     return (gl_impl == gfx::kGLImplementationDesktopGL ||
    135             gl_impl == gfx::kGLImplementationEGLGLES2);
    136   }
    137 
    138   void RunTest(const std::string& name) {
    139 #if defined(USE_AURA)
    140     if (!HasFlag(kUseGpu)) {
    141       printf("Test skipped, Aura always runs with GPU\n");
    142       return;
    143     }
    144 #endif
    145 #if defined(OS_WIN)
    146     if (HasFlag(kUseGpu) && HasFlag(kIsGpuCanvasTest) &&
    147         base::win::OSInfo::GetInstance()->version() == base::win::VERSION_XP) {
    148       // crbug.com/128208
    149       LOG(WARNING) << "Test skipped: GPU canvas tests do not run on XP.";
    150       return;
    151     }
    152 #endif
    153 
    154     if (HasFlag(kUseGpu) && !IsGpuAvailable()) {
    155       printf("Test skipped: requires gpu. Pass --enable-gpu on the command "
    156              "line if use of GPU is desired.\n");
    157       return;
    158     }
    159 
    160     // Verify flag combinations.
    161     ASSERT_TRUE(HasFlag(kUseGpu) || !HasFlag(kForceGpuComposited));
    162     ASSERT_TRUE(!HasFlag(kUseGpu) || IsGpuAvailable());
    163 
    164     base::FilePath test_path = GetDataPath(name);
    165     ASSERT_TRUE(base::DirectoryExists(test_path))
    166         << "Missing test directory: " << test_path.value();
    167 
    168     test_path = test_path.Append(FILE_PATH_LITERAL("test.html"));
    169 
    170     scoped_refptr<TabProxy> tab(GetActiveTab());
    171     ASSERT_TRUE(tab.get());
    172 
    173     // TODO(jbates): remove this check when ref builds are updated.
    174     if (!HasFlag(kUseReferenceBuild))
    175       ASSERT_TRUE(automation()->BeginTracing("test_gpu"));
    176 
    177     if (HasFlag(kHasRedirect)) {
    178       // If the test file is known to contain an html redirect, we must block
    179       // until the second navigation is complete and reacquire the active tab
    180       // in order to avoid a race condition.
    181       // If the following assertion is triggered due to a timeout, it is
    182       // possible that the current test does not re-direct and therefore should
    183       // not have the kHasRedirect flag turned on.
    184       ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS,
    185         tab->NavigateToURLBlockUntilNavigationsComplete(
    186         net::FilePathToFileURL(test_path), 2));
    187       tab = GetActiveTab();
    188       ASSERT_TRUE(tab.get());
    189     } else {
    190       ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS,
    191         tab->NavigateToURL(net::FilePathToFileURL(test_path)));
    192     }
    193 
    194     // Block until initialization completes
    195     // If the following assertion fails intermittently, it could be due to a
    196     // race condition caused by an html redirect. If that is the case, verify
    197     // that flag kHasRedirect is enabled for the current test.
    198     ASSERT_TRUE(WaitUntilJavaScriptCondition(
    199         tab.get(),
    200         std::wstring(),
    201         L"window.domAutomationController.send(__initialized);",
    202         TestTimeouts::large_test_timeout()));
    203 
    204     if (HasFlag(kForceGpuComposited)) {
    205       ASSERT_TRUE(tab->NavigateToURLAsync(
    206         GURL("javascript:__make_body_composited();")));
    207     }
    208 
    209     // Start the tests.
    210     ASSERT_TRUE(tab->NavigateToURLAsync(GURL("javascript:__start_all();")));
    211 
    212     // Block until the tests completes.
    213     ASSERT_TRUE(WaitUntilJavaScriptCondition(
    214         tab.get(),
    215         std::wstring(),
    216         L"window.domAutomationController.send(!__running_all);",
    217         TestTimeouts::large_test_timeout()));
    218 
    219     // TODO(jbates): remove this check when ref builds are updated.
    220     if (!HasFlag(kUseReferenceBuild)) {
    221       std::string json_events;
    222       ASSERT_TRUE(automation()->EndTracing(&json_events));
    223 
    224       bool did_run_on_gpu = DidRunOnGpu(json_events);
    225       bool expect_gpu = HasFlag(kUseGpu);
    226       EXPECT_EQ(expect_gpu, did_run_on_gpu);
    227     }
    228 
    229     // Read out the results.
    230     std::wstring json;
    231     ASSERT_TRUE(tab->ExecuteAndExtractString(
    232         std::wstring(),
    233         L"window.domAutomationController.send("
    234         L"JSON.stringify(__calc_results_total()));",
    235         &json));
    236 
    237     std::map<std::string, std::string> results;
    238     ASSERT_TRUE(JsonDictionaryToMap(WideToUTF8(json), &results));
    239 
    240     ASSERT_TRUE(results.find("mean") != results.end());
    241     ASSERT_TRUE(results.find("sigma") != results.end());
    242     ASSERT_TRUE(results.find("gestures") != results.end());
    243     ASSERT_TRUE(results.find("means") != results.end());
    244     ASSERT_TRUE(results.find("sigmas") != results.end());
    245 
    246     std::string trace_name = "interval" + GetSuffixForTestFlags();
    247     printf("GESTURES %s: %s= [%s] [%s] [%s]\n", name.c_str(),
    248                                                 trace_name.c_str(),
    249                                                 results["gestures"].c_str(),
    250                                                 results["means"].c_str(),
    251                                                 results["sigmas"].c_str());
    252 
    253     std::string mean_and_error = results["mean"] + "," + results["sigma"];
    254     perf_test::PrintResultMeanAndError(name,
    255                                        std::string(),
    256                                        trace_name,
    257                                        mean_and_error,
    258                                        "milliseconds-per-frame",
    259                                        true);
    260 
    261     // Navigate back to NTP so that we can quit without timing out during the
    262     // wait-for-idle stage in test framework.
    263     EXPECT_EQ(tab->GoBack(), AUTOMATION_MSG_NAVIGATION_SUCCESS);
    264   }
    265 };
    266 
    267 // Must use a different class name to avoid test instantiation conflicts
    268 // with FrameRateTest. An alias is good enough. The alias names must match
    269 // the pattern FrameRate*Test* for them to get picked up by the test bots.
    270 typedef FrameRateTest FrameRateCompositingTest;
    271 
    272 // Tests that trigger compositing with a -webkit-translateZ(0)
    273 #define FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(content) \
    274 TEST_P(FrameRateCompositingTest, content) { \
    275   RunTest(#content); \
    276 }
    277 
    278 INSTANTIATE_TEST_CASE_P(, FrameRateCompositingTest, ::testing::Values(
    279                         0,
    280                         kUseGpu | kForceGpuComposited,
    281                         kUseReferenceBuild,
    282                         kUseReferenceBuild | kUseGpu | kForceGpuComposited));
    283 
    284 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(blank);
    285 FRAME_RATE_TEST_WITH_AND_WITHOUT_ACCELERATED_COMPOSITING(googleblog);
    286 
    287 typedef FrameRateTest FrameRateNoVsyncCanvasInternalTest;
    288 
    289 // Tests for animated 2D canvas content with and without disabling vsync
    290 #define INTERNAL_FRAME_RATE_TEST_CANVAS_WITH_AND_WITHOUT_NOVSYNC(content) \
    291 TEST_P(FrameRateNoVsyncCanvasInternalTest, content) { \
    292   RunTest(#content); \
    293 }
    294 
    295 INSTANTIATE_TEST_CASE_P(, FrameRateNoVsyncCanvasInternalTest, ::testing::Values(
    296     kInternal | kHasRedirect,
    297     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu,
    298     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu | kDisableVsync,
    299     kUseReferenceBuild | kInternal | kHasRedirect,
    300     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu,
    301     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu |
    302         kDisableVsync));
    303 
    304 INTERNAL_FRAME_RATE_TEST_CANVAS_WITH_AND_WITHOUT_NOVSYNC(fishbowl)
    305 
    306 typedef FrameRateTest FrameRateGpuCanvasInternalTest;
    307 
    308 // Tests for animated 2D canvas content to be tested only with GPU
    309 // acceleration.
    310 // tests are run with and without Vsync
    311 #define INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(content) \
    312 TEST_P(FrameRateGpuCanvasInternalTest, content) { \
    313   RunTest(#content); \
    314 }
    315 
    316 INSTANTIATE_TEST_CASE_P(, FrameRateGpuCanvasInternalTest, ::testing::Values(
    317     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu,
    318     kIsGpuCanvasTest | kInternal | kHasRedirect | kUseGpu | kDisableVsync,
    319     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu,
    320     kIsGpuCanvasTest | kUseReferenceBuild | kInternal | kHasRedirect | kUseGpu |
    321         kDisableVsync));
    322 
    323 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(fireflies)
    324 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(FishIE)
    325 INTERNAL_FRAME_RATE_TEST_CANVAS_GPU(speedreading)
    326 
    327 }  // namespace
    328