Home | History | Annotate | Download | only in perf 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/environment.h"
      6 #include "base/file_util.h"
      7 #include "base/files/file_enumerator.h"
      8 #include "base/path_service.h"
      9 #include "base/strings/string_number_conversions.h"
     10 #include "base/strings/string_split.h"
     11 #include "base/strings/string_util.h"
     12 #include "base/strings/stringprintf.h"
     13 #include "base/strings/utf_string_conversions.h"
     14 #include "base/sys_info.h"
     15 #include "base/test/test_file_util.h"
     16 #include "base/test/test_timeouts.h"
     17 #include "base/time/time.h"
     18 #include "chrome/common/chrome_constants.h"
     19 #include "chrome/common/chrome_paths.h"
     20 #include "chrome/common/chrome_switches.h"
     21 #include "chrome/common/env_vars.h"
     22 #include "chrome/test/automation/automation_proxy.h"
     23 #include "chrome/test/base/test_switches.h"
     24 #include "chrome/test/base/testing_profile.h"
     25 #include "chrome/test/base/ui_test_utils.h"
     26 #include "chrome/test/perf/perf_test.h"
     27 #include "chrome/test/perf/perf_ui_test_suite.h"
     28 #include "chrome/test/ui/ui_perf_test.h"
     29 #include "content/public/common/content_switches.h"
     30 #include "net/base/net_util.h"
     31 
     32 using base::TimeDelta;
     33 using base::TimeTicks;
     34 
     35 namespace {
     36 
     37 class StartupTest : public UIPerfTest {
     38  public:
     39   StartupTest() {
     40     show_window_ = true;
     41   }
     42   virtual void SetUp() {
     43     collect_profiling_stats_ = false;
     44   }
     45   virtual void TearDown() {}
     46 
     47   enum TestColdness {
     48     WARM,
     49     COLD
     50   };
     51 
     52   enum TestImportance {
     53     NOT_IMPORTANT,
     54     IMPORTANT
     55   };
     56 
     57   // Load a file on startup rather than about:blank.  This tests a longer
     58   // startup path, including resource loading.
     59   void SetUpWithFileURL() {
     60     const base::FilePath file_url = ui_test_utils::GetTestFilePath(
     61         base::FilePath(base::FilePath::kCurrentDirectory),
     62         base::FilePath(FILE_PATH_LITERAL("simple.html")));
     63     ASSERT_TRUE(base::PathExists(file_url));
     64     launch_arguments_.AppendArgPath(file_url);
     65   }
     66 
     67   // Setup the command line arguments to capture profiling data for tasks.
     68   void SetUpWithProfiling() {
     69     profiling_file_ = ui_test_utils::GetTestFilePath(
     70         base::FilePath(base::FilePath::kCurrentDirectory),
     71         base::FilePath(FILE_PATH_LITERAL("task_profile.json")));
     72     launch_arguments_.AppendSwitchPath(switches::kProfilingOutputFile,
     73                                        profiling_file_);
     74     collect_profiling_stats_ = true;
     75   }
     76 
     77   // Load a complex html file on startup represented by |which_tab|.
     78   void SetUpWithComplexFileURL(unsigned int which_tab) {
     79     const char* const kTestPageCyclerDomains[] = {
     80         "www.google.com", "www.nytimes.com",
     81         "www.yahoo.com", "espn.go.com", "www.amazon.com"
     82     };
     83     unsigned int which_el = which_tab % arraysize(kTestPageCyclerDomains);
     84     const char* this_domain = kTestPageCyclerDomains[which_el];
     85 
     86     base::FilePath page_cycler_path;
     87     PathService::Get(base::DIR_SOURCE_ROOT, &page_cycler_path);
     88     page_cycler_path = page_cycler_path.AppendASCII("data")
     89         .AppendASCII("page_cycler").AppendASCII("moz")
     90         .AppendASCII(this_domain).AppendASCII("index.html");
     91     GURL file_url = net::FilePathToFileURL(page_cycler_path).Resolve("?skip");
     92     launch_arguments_.AppendArg(file_url.spec());
     93   }
     94 
     95   // Use the given profile in the test data extensions/profiles dir.  This tests
     96   // startup with extensions installed.
     97   void SetUpWithExtensionsProfile(const char* profile) {
     98     base::FilePath data_dir;
     99     PathService::Get(chrome::DIR_TEST_DATA, &data_dir);
    100     data_dir = data_dir.AppendASCII("extensions").AppendASCII("profiles").
    101         AppendASCII(profile);
    102     set_template_user_data(data_dir);
    103   }
    104 
    105   // Rewrite the preferences file to point to the proper image directory.
    106   virtual void SetUpProfile() OVERRIDE {
    107     UIPerfTest::SetUpProfile();
    108     if (profile_type_ != PerfUITestSuite::COMPLEX_THEME)
    109       return;
    110 
    111     const base::FilePath pref_template_path(user_data_dir().
    112         AppendASCII("Default").
    113         AppendASCII("PreferencesTemplate"));
    114     const base::FilePath pref_path(user_data_dir().
    115         AppendASCII(TestingProfile::kTestUserProfileDir).
    116         AppendASCII("Preferences"));
    117 
    118     // Read in preferences template.
    119     std::string pref_string;
    120     EXPECT_TRUE(file_util::ReadFileToString(pref_template_path, &pref_string));
    121     string16 format_string = ASCIIToUTF16(pref_string);
    122 
    123     // Make sure temp directory has the proper format for writing to prefs file.
    124 #if defined(OS_POSIX)
    125     std::wstring user_data_dir_w(ASCIIToWide(user_data_dir().value()));
    126 #elif defined(OS_WIN)
    127     std::wstring user_data_dir_w(user_data_dir().value());
    128     // In Windows, the FilePath will write '\' for the path separators; change
    129     // these to a separator that won't trigger escapes.
    130     std::replace(user_data_dir_w.begin(),
    131                  user_data_dir_w.end(), '\\', '/');
    132 #endif
    133 
    134     // Rewrite prefs file.
    135     std::vector<string16> subst;
    136     subst.push_back(WideToUTF16(user_data_dir_w));
    137     const std::string prefs_string =
    138         UTF16ToASCII(ReplaceStringPlaceholders(format_string, subst, NULL));
    139     EXPECT_TRUE(file_util::WriteFile(pref_path, prefs_string.c_str(),
    140                                      prefs_string.size()));
    141     file_util::EvictFileFromSystemCache(pref_path);
    142   }
    143 
    144   // Runs a test which loads |tab_count| tabs on startup, either as command line
    145   // arguments or, if |restore_session| is true, by using session restore.
    146   // |nth_timed_tab|, if non-zero, will measure time to load the first n+1 tabs.
    147   void RunPerfTestWithManyTabs(const char* graph, const char* trace,
    148                                int tab_count, int nth_timed_tab,
    149                                bool restore_session);
    150 
    151   void RunStartupTest(const char* graph, const char* trace,
    152                       TestColdness test_cold, TestImportance test_importance,
    153                       PerfUITestSuite::ProfileType profile_type,
    154                       int num_tabs, int nth_timed_tab) {
    155     bool important = (test_importance == IMPORTANT);
    156     profile_type_ = profile_type;
    157 
    158     // Sets the profile data for the run.  For now, this is only used for
    159     // the non-default themes test.
    160     if (profile_type != PerfUITestSuite::DEFAULT_THEME) {
    161       set_template_user_data(
    162           PerfUITestSuite::GetPathForProfileType(profile_type));
    163     }
    164 
    165 #if defined(NDEBUG)
    166     const int kNumCyclesMax = 20;
    167 #else
    168     // Debug builds are too slow and we can't run that many cycles in a
    169     // reasonable amount of time.
    170     const int kNumCyclesMax = 10;
    171 #endif
    172     int numCycles = kNumCyclesMax;
    173     scoped_ptr<base::Environment> env(base::Environment::Create());
    174     std::string numCyclesEnv;
    175     if (env->GetVar(env_vars::kStartupTestsNumCycles, &numCyclesEnv) &&
    176         base::StringToInt(numCyclesEnv, &numCycles)) {
    177       if (numCycles <= kNumCyclesMax) {
    178         VLOG(1) << env_vars::kStartupTestsNumCycles
    179                 << " set in environment, so setting numCycles to " << numCycles;
    180       } else {
    181         VLOG(1) << env_vars::kStartupTestsNumCycles
    182                 << " is higher than the max, setting numCycles to "
    183                 << kNumCyclesMax;
    184         numCycles = kNumCyclesMax;
    185       }
    186     }
    187 
    188     struct TimingInfo {
    189       TimeDelta end_to_end;
    190       float first_start_ms;
    191       float last_stop_ms;
    192       float first_stop_ms;
    193       float nth_tab_stop_ms;
    194     };
    195     TimingInfo timings[kNumCyclesMax];
    196 
    197     CommandLine launch_arguments_without_trace_file(launch_arguments_);
    198     for (int i = 0; i < numCycles; ++i) {
    199       if (test_cold == COLD) {
    200         base::FilePath dir_app;
    201         ASSERT_TRUE(PathService::Get(chrome::DIR_APP, &dir_app));
    202 
    203         base::FilePath chrome_exe(dir_app.Append(GetExecutablePath()));
    204         ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_exe));
    205 #if defined(OS_WIN)
    206         // chrome.dll is windows specific.
    207         base::FilePath chrome_dll(
    208             dir_app.Append(FILE_PATH_LITERAL("chrome.dll")));
    209         ASSERT_TRUE(base::EvictFileFromSystemCacheWithRetry(chrome_dll));
    210 #endif
    211       }
    212       UITest::SetUp();
    213       TimeTicks end_time = TimeTicks::Now();
    214 
    215       if (num_tabs > 0) {
    216         float min_start;
    217         float max_stop;
    218         std::vector<float> times;
    219         scoped_refptr<BrowserProxy> browser_proxy(
    220             automation()->GetBrowserWindow(0));
    221         ASSERT_TRUE(browser_proxy.get());
    222 
    223         if (browser_proxy->GetInitialLoadTimes(
    224               TestTimeouts::action_max_timeout(),
    225               &min_start,
    226               &max_stop,
    227               &times) &&
    228             !times.empty()) {
    229           ASSERT_LT(nth_timed_tab, num_tabs);
    230           ASSERT_EQ(times.size(), static_cast<size_t>(num_tabs));
    231           timings[i].first_start_ms = min_start;
    232           timings[i].last_stop_ms = max_stop;
    233           timings[i].first_stop_ms = times[0];
    234           timings[i].nth_tab_stop_ms = times[nth_timed_tab];
    235         } else {
    236           // Browser might not support initial load times.
    237           // Only use end-to-end time for this test.
    238           num_tabs = 0;
    239         }
    240       }
    241       timings[i].end_to_end = end_time - browser_launch_time();
    242       UITest::TearDown();
    243 
    244       if (i == 0) {
    245         // Re-use the profile data after first run so that the noise from
    246         // creating databases doesn't impact all the runs.
    247         clear_profile_ = false;
    248         // Clear template_user_data_ so we don't try to copy it over each time
    249         // through.
    250         set_template_user_data(base::FilePath());
    251       }
    252     }
    253 
    254     std::string times;
    255     for (int i = 0; i < numCycles; ++i) {
    256       base::StringAppendF(&times,
    257                           "%.2f,",
    258                           timings[i].end_to_end.InMillisecondsF());
    259     }
    260     perf_test::PrintResultList(
    261         graph, std::string(), trace, times, "ms", important);
    262 
    263     if (num_tabs > 0) {
    264       std::string name_base = trace;
    265       std::string name;
    266 
    267       times.clear();
    268       name = name_base + "-start";
    269       for (int i = 0; i < numCycles; ++i)
    270         base::StringAppendF(&times, "%.2f,", timings[i].first_start_ms);
    271       perf_test::PrintResultList(
    272           graph, std::string(), name.c_str(), times, "ms", important);
    273 
    274       times.clear();
    275       name = name_base + "-first";
    276       for (int i = 0; i < numCycles; ++i)
    277         base::StringAppendF(&times, "%.2f,", timings[i].first_stop_ms);
    278       perf_test::PrintResultList(
    279           graph, std::string(), name.c_str(), times, "ms", important);
    280 
    281       if (nth_timed_tab > 0) {
    282         // Display only the time necessary to load the first n tabs.
    283         times.clear();
    284         name = name_base + "-" + base::IntToString(nth_timed_tab);
    285         for (int i = 0; i < numCycles; ++i)
    286           base::StringAppendF(&times, "%.2f,", timings[i].nth_tab_stop_ms);
    287         perf_test::PrintResultList(
    288             graph, std::string(), name.c_str(), times, "ms", important);
    289       }
    290 
    291       if (num_tabs > 1) {
    292         // Display the time necessary to load all of the tabs.
    293         times.clear();
    294         name = name_base + "-all";
    295         for (int i = 0; i < numCycles; ++i)
    296           base::StringAppendF(&times, "%.2f,", timings[i].last_stop_ms);
    297         perf_test::PrintResultList(
    298             graph, std::string(), name.c_str(), times, "ms", important);
    299       }
    300     }
    301   }
    302 
    303   base::FilePath profiling_file_;
    304   bool collect_profiling_stats_;
    305   std::string trace_file_prefix_;
    306 
    307   // Are we using a profile with a complex theme?
    308   PerfUITestSuite::ProfileType profile_type_;
    309 };
    310 
    311 TEST_F(StartupTest, PerfWarm) {
    312   RunStartupTest("warm", "t", WARM, IMPORTANT,
    313                  PerfUITestSuite::DEFAULT_THEME, 0, 0);
    314 }
    315 
    316 TEST_F(StartupTest, PerfReferenceWarm) {
    317   UseReferenceBuild();
    318   RunStartupTest("warm", "t_ref", WARM, IMPORTANT,
    319                  PerfUITestSuite::DEFAULT_THEME, 0, 0);
    320 }
    321 
    322 // TODO(mpcomplete): Should we have reference timings for all these?
    323 
    324 // dominich: Disabling as per http://crbug.com/100900.
    325 #if defined(OS_WIN)
    326 #define MAYBE_PerfCold DISABLED_PerfCold
    327 #else
    328 #define MAYBE_PerfCold PerfCold
    329 #endif
    330 
    331 TEST_F(StartupTest, MAYBE_PerfCold) {
    332   RunStartupTest("cold", "t", COLD, NOT_IMPORTANT,
    333                  PerfUITestSuite::DEFAULT_THEME, 0, 0);
    334 }
    335 
    336 void StartupTest::RunPerfTestWithManyTabs(const char* graph, const char* trace,
    337                                           int tab_count, int nth_timed_tab,
    338                                           bool restore_session) {
    339   // Initialize session with |tab_count| tabs.
    340   for (int i = 0; i < tab_count; ++i)
    341     SetUpWithComplexFileURL(i);
    342 
    343   if (restore_session) {
    344     // Start the browser with these urls so we can save the session and exit.
    345     UITest::SetUp();
    346     // Set flags to ensure profile is saved and can be restored.
    347 #if defined(OS_CHROMEOS) || defined(OS_MACOSX)
    348     set_shutdown_type(ProxyLauncher::USER_QUIT);
    349 #endif
    350     clear_profile_ = false;
    351     // Quit and set flags to restore session.
    352     UITest::TearDown();
    353 
    354     // Clear all arguments for session restore, or the number of open tabs
    355     // will grow with each restore.
    356     CommandLine new_launch_arguments = CommandLine(CommandLine::NO_PROGRAM);
    357     // Keep the branding switch if using a reference build.
    358     if (launch_arguments_.HasSwitch(switches::kEnableChromiumBranding)) {
    359       new_launch_arguments.AppendSwitch(switches::kEnableChromiumBranding);
    360     }
    361     // The session will be restored once per cycle for numCycles test cycles,
    362     // and each time, UITest::SetUp will wait for |tab_count| tabs to
    363     // finish loading.
    364     new_launch_arguments.AppendSwitchASCII(switches::kRestoreLastSession,
    365                                         base::IntToString(tab_count));
    366     launch_arguments_ = new_launch_arguments;
    367   }
    368   RunStartupTest(graph, trace, WARM, NOT_IMPORTANT,
    369                  PerfUITestSuite::DEFAULT_THEME, tab_count, nth_timed_tab);
    370 }
    371 
    372 // http://crbug.com/101591
    373 #if defined(OS_WIN) && !defined(NDEBUG)
    374 #define MAYBE_PerfFewTabs DISABLED_PerfFewTabs
    375 #else
    376 #define MAYBE_PerfFewTabs PerfFewTabs
    377 #endif
    378 
    379 TEST_F(StartupTest, MAYBE_PerfFewTabs) {
    380   RunPerfTestWithManyTabs("few_tabs", "cmdline", 5, 2, false);
    381 }
    382 
    383 TEST_F(StartupTest, PerfFewTabsReference) {
    384   UseReferenceBuild();
    385   RunPerfTestWithManyTabs("few_tabs", "cmdline-ref", 5, 2, false);
    386 }
    387 
    388 TEST_F(StartupTest, PerfRestoreFewTabs) {
    389   RunPerfTestWithManyTabs("few_tabs", "restore", 5, 2, true);
    390 }
    391 
    392 TEST_F(StartupTest, PerfRestoreFewTabsReference) {
    393   UseReferenceBuild();
    394   RunPerfTestWithManyTabs("few_tabs", "restore-ref", 5, 2, true);
    395 }
    396 
    397 #if defined(OS_MACOSX)
    398 // http://crbug.com/46609
    399 #define MAYBE_PerfSeveralTabsReference DISABLED_PerfSeveralTabsReference
    400 #define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs
    401 // http://crbug.com/52858
    402 #define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs
    403 #define MAYBE_PerfExtensionContentScript50 DISABLED_PerfExtensionContentScript50
    404 #elif defined(OS_WIN)
    405 // http://crbug.com/46609
    406 #if !defined(NDEBUG)
    407 // This test is disabled on Windows Debug. See bug http://crbug.com/132279
    408 #define MAYBE_PerfRestoreSeveralTabs DISABLED_PerfRestoreSeveralTabs
    409 #else
    410 #define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs
    411 #endif  // !defined(NDEBUG)
    412 // http://crbug.com/102584
    413 #define MAYBE_PerfSeveralTabs DISABLED_PerfSeveralTabs
    414 #define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference
    415 #define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50
    416 #else
    417 #define MAYBE_PerfSeveralTabsReference PerfSeveralTabsReference
    418 #define MAYBE_PerfSeveralTabs PerfSeveralTabs
    419 #define MAYBE_PerfRestoreSeveralTabs PerfRestoreSeveralTabs
    420 #define MAYBE_PerfExtensionContentScript50 PerfExtensionContentScript50
    421 #endif
    422 
    423 // http://crbug.com/99604
    424 #if defined(OS_WIN) && !defined(NDEBUG)
    425 #define MAYBE_PerfComplexTheme DISABLED_PerfComplexTheme
    426 #else
    427 #define MAYBE_PerfComplexTheme PerfComplexTheme
    428 #endif
    429 
    430 TEST_F(StartupTest, MAYBE_PerfSeveralTabs) {
    431   RunPerfTestWithManyTabs("several_tabs", "cmdline", 10, 4, false);
    432 }
    433 
    434 TEST_F(StartupTest, MAYBE_PerfSeveralTabsReference) {
    435   UseReferenceBuild();
    436   RunPerfTestWithManyTabs("several_tabs", "cmdline-ref", 10, 4, false);
    437 }
    438 
    439 TEST_F(StartupTest, MAYBE_PerfRestoreSeveralTabs) {
    440   RunPerfTestWithManyTabs("several_tabs", "restore", 10, 4, true);
    441 }
    442 
    443 TEST_F(StartupTest, PerfRestoreSeveralTabsReference) {
    444   UseReferenceBuild();
    445   RunPerfTestWithManyTabs("several_tabs", "restore-ref", 10, 4, true);
    446 }
    447 
    448 TEST_F(StartupTest, PerfExtensionEmpty) {
    449   SetUpWithFileURL();
    450   SetUpWithExtensionsProfile("empty");
    451   RunStartupTest("warm", "extension_empty", WARM, NOT_IMPORTANT,
    452                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
    453 }
    454 
    455 TEST_F(StartupTest, PerfExtensionContentScript1) {
    456   SetUpWithFileURL();
    457   SetUpWithExtensionsProfile("content_scripts1");
    458   RunStartupTest("warm", "extension_content_scripts1", WARM, NOT_IMPORTANT,
    459                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
    460 }
    461 
    462 TEST_F(StartupTest, MAYBE_PerfExtensionContentScript50) {
    463   SetUpWithFileURL();
    464   SetUpWithExtensionsProfile("content_scripts50");
    465   RunStartupTest("warm", "extension_content_scripts50", WARM, NOT_IMPORTANT,
    466                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
    467 }
    468 
    469 TEST_F(StartupTest, MAYBE_PerfComplexTheme) {
    470   RunStartupTest("warm", "t-theme", WARM, NOT_IMPORTANT,
    471                  PerfUITestSuite::COMPLEX_THEME, 0, 0);
    472 }
    473 
    474 TEST_F(StartupTest, ProfilingScript1) {
    475   SetUpWithFileURL();
    476   SetUpWithProfiling();
    477   RunStartupTest("warm", "profiling_scripts1", WARM, NOT_IMPORTANT,
    478                  PerfUITestSuite::DEFAULT_THEME, 1, 0);
    479 }
    480 
    481 }  // namespace
    482