Home | History | Annotate | Download | only in perf 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/command_line.h"
      6 #include "base/file_util.h"
      7 #include "base/files/file_path.h"
      8 #include "base/json/json_string_value_serializer.h"
      9 #include "base/memory/scoped_ptr.h"
     10 #include "base/path_service.h"
     11 #include "base/strings/string_split.h"
     12 #include "base/strings/string_util.h"
     13 #include "base/strings/utf_string_conversions.h"
     14 #include "base/test/test_timeouts.h"
     15 #include "base/values.h"
     16 #include "chrome/common/chrome_paths.h"
     17 #include "chrome/common/chrome_switches.h"
     18 #include "chrome/test/automation/tab_proxy.h"
     19 #include "chrome/test/ui/ui_test.h"
     20 #include "net/base/net_util.h"
     21 #include "url/gurl.h"
     22 
     23 namespace {
     24 
     25 static const base::FilePath::CharType kBaseUrl[] =
     26     FILE_PATH_LITERAL("http://localhost:8000/");
     27 
     28 static const base::FilePath::CharType kTestDirectory[] =
     29     FILE_PATH_LITERAL("dom_checker/");
     30 
     31 static const base::FilePath::CharType kStartFile[] =
     32     FILE_PATH_LITERAL("dom_checker.html");
     33 
     34 const char kRunDomCheckerTest[] = "run-dom-checker-test";
     35 
     36 class DomCheckerTest : public UITest {
     37  public:
     38   typedef std::list<std::string> ResultsList;
     39   typedef std::set<std::string> ResultsSet;
     40 
     41   DomCheckerTest() {
     42     dom_automation_enabled_ = true;
     43     enable_file_cookies_ = false;
     44     show_window_ = true;
     45     launch_arguments_.AppendSwitch(switches::kDisablePopupBlocking);
     46   }
     47 
     48   void RunTest(bool use_http, ResultsList* new_passes,
     49                ResultsList* new_failures) {
     50     int test_count = 0;
     51     ResultsSet expected_failures, current_failures;
     52 
     53     std::string failures_file = use_http ?
     54 #if defined(OS_MACOSX)
     55         "expected_failures-http.txt" : "expected_failures_mac-file.txt";
     56 #elif defined(OS_LINUX)
     57         "expected_failures-http.txt" : "expected_failures_linux-file.txt";
     58 #elif defined(OS_WIN)
     59         "expected_failures-http.txt" : "expected_failures_win-file.txt";
     60 #else
     61         "" : "";
     62 #endif
     63 
     64     ASSERT_TRUE(GetExpectedFailures(failures_file, &expected_failures));
     65 
     66     RunDomChecker(use_http, &test_count, &current_failures);
     67     printf("\nTests run: %d\n", test_count);
     68 
     69     // Compute the list of new passes and failures.
     70     CompareSets(current_failures, expected_failures, new_passes);
     71     CompareSets(expected_failures, current_failures, new_failures);
     72   }
     73 
     74   void PrintResults(const ResultsList& new_passes,
     75                     const ResultsList& new_failures) {
     76     PrintResults(new_failures, "new tests failing", true);
     77     PrintResults(new_passes, "new tests passing", false);
     78   }
     79 
     80  private:
     81   void PrintResults(const ResultsList& results, const char* message,
     82                     bool add_failure) {
     83     if (!results.empty()) {
     84       if (add_failure)
     85         ADD_FAILURE();
     86 
     87       printf("%s:\n", message);
     88       ResultsList::const_iterator it = results.begin();
     89       for (; it != results.end(); ++it)
     90         printf("  %s\n", it->c_str());
     91       printf("\n");
     92     }
     93   }
     94 
     95   // Find the elements of "b" that are not in "a".
     96   void CompareSets(const ResultsSet& a, const ResultsSet& b,
     97                    ResultsList* only_in_b) {
     98     ResultsSet::const_iterator it = b.begin();
     99     for (; it != b.end(); ++it) {
    100       if (a.find(*it) == a.end())
    101         only_in_b->push_back(*it);
    102     }
    103   }
    104 
    105   // Return the path to the DOM checker directory on the local filesystem.
    106   base::FilePath GetDomCheckerDir() {
    107     base::FilePath test_dir;
    108     PathService::Get(chrome::DIR_TEST_DATA, &test_dir);
    109     return test_dir.AppendASCII("dom_checker");
    110   }
    111 
    112   bool ReadExpectedResults(const std::string& failures_file,
    113                            std::string* results) {
    114     base::FilePath results_path = GetDomCheckerDir();
    115     results_path = results_path.AppendASCII(failures_file);
    116     return file_util::ReadFileToString(results_path, results);
    117   }
    118 
    119   void ParseExpectedFailures(const std::string& input, ResultsSet* output) {
    120     if (input.empty())
    121       return;
    122 
    123     std::vector<std::string> tokens;
    124     base::SplitString(input, '\n', &tokens);
    125 
    126     std::vector<std::string>::const_iterator it = tokens.begin();
    127     for (; it != tokens.end(); ++it) {
    128       // Allow comments (lines that start with #).
    129       if (it->length() > 0 && it->at(0) != '#')
    130         output->insert(*it);
    131     }
    132   }
    133 
    134   bool GetExpectedFailures(const std::string& failures_file,
    135                            ResultsSet* expected_failures) {
    136     std::string expected_failures_text;
    137     bool have_expected_results = ReadExpectedResults(failures_file,
    138                                                      &expected_failures_text);
    139     if (!have_expected_results)
    140       return false;
    141     ParseExpectedFailures(expected_failures_text, expected_failures);
    142     return true;
    143   }
    144 
    145   bool WaitUntilTestCompletes(TabProxy* tab) {
    146     return WaitUntilJavaScriptCondition(
    147         tab,
    148         std::wstring(),
    149         L"window.domAutomationController.send(automation.IsDone());",
    150         TestTimeouts::large_test_timeout());
    151   }
    152 
    153   bool GetTestCount(TabProxy* tab, int* test_count) {
    154     return tab->ExecuteAndExtractInt(
    155         std::wstring(),
    156         L"window.domAutomationController.send(automation.GetTestCount());",
    157         test_count);
    158   }
    159 
    160   bool GetTestsFailed(TabProxy* tab, ResultsSet* tests_failed) {
    161     std::wstring json_wide;
    162     bool succeeded = tab->ExecuteAndExtractString(
    163         std::wstring(),
    164         L"window.domAutomationController.send("
    165         L"    JSON.stringify(automation.GetFailures()));",
    166         &json_wide);
    167 
    168     // Note that we don't use ASSERT_TRUE here (and in some other places) as it
    169     // doesn't work inside a function with a return type other than void.
    170     EXPECT_TRUE(succeeded);
    171     if (!succeeded)
    172       return false;
    173 
    174     std::string json = WideToUTF8(json_wide);
    175     JSONStringValueSerializer deserializer(json);
    176     scoped_ptr<Value> value(deserializer.Deserialize(NULL, NULL));
    177 
    178     EXPECT_TRUE(value.get());
    179     if (!value.get())
    180       return false;
    181 
    182     EXPECT_TRUE(value->IsType(Value::TYPE_LIST));
    183     if (!value->IsType(Value::TYPE_LIST))
    184       return false;
    185 
    186     ListValue* list_value = static_cast<ListValue*>(value.get());
    187 
    188     // The parsed JSON object will be an array of strings, each of which is a
    189     // test failure. Add those strings to the results set.
    190     ListValue::const_iterator it = list_value->begin();
    191     for (; it != list_value->end(); ++it) {
    192       EXPECT_TRUE((*it)->IsType(Value::TYPE_STRING));
    193       if ((*it)->IsType(Value::TYPE_STRING)) {
    194         std::string test_name;
    195         succeeded = (*it)->GetAsString(&test_name);
    196         EXPECT_TRUE(succeeded);
    197         if (succeeded)
    198           tests_failed->insert(test_name);
    199       }
    200     }
    201 
    202     return true;
    203   }
    204 
    205   void RunDomChecker(bool use_http, int* test_count, ResultsSet* tests_failed) {
    206     GURL test_url;
    207     base::FilePath::StringType start_file(kStartFile);
    208     if (use_http) {
    209       base::FilePath::StringType test_directory(kTestDirectory);
    210       base::FilePath::StringType url_string(kBaseUrl);
    211       url_string.append(test_directory);
    212       url_string.append(start_file);
    213       test_url = GURL(url_string);
    214     } else {
    215       base::FilePath test_path = GetDomCheckerDir();
    216       test_path = test_path.Append(start_file);
    217       test_url = net::FilePathToFileURL(test_path);
    218     }
    219 
    220     scoped_refptr<TabProxy> tab(GetActiveTab());
    221     ASSERT_TRUE(tab.get());
    222     ASSERT_EQ(AUTOMATION_MSG_NAVIGATION_SUCCESS, tab->NavigateToURL(test_url));
    223 
    224     // Wait for the test to finish.
    225     ASSERT_TRUE(WaitUntilTestCompletes(tab.get()));
    226 
    227     // Get the test results.
    228     ASSERT_TRUE(GetTestCount(tab.get(), test_count));
    229     ASSERT_TRUE(GetTestsFailed(tab.get(), tests_failed));
    230     ASSERT_GT(*test_count, 0);
    231   }
    232 
    233   DISALLOW_COPY_AND_ASSIGN(DomCheckerTest);
    234 };
    235 
    236 // Always fails, see but http://crbug.com/21321
    237 TEST_F(DomCheckerTest, DISABLED_File) {
    238   if (!CommandLine::ForCurrentProcess()->HasSwitch(kRunDomCheckerTest))
    239     return;
    240 
    241   ResultsList new_passes, new_failures;
    242   RunTest(false, &new_passes, &new_failures);
    243   PrintResults(new_passes, new_failures);
    244 }
    245 
    246 // This test was previously failing because it was looking for an
    247 // expected results file that didn't exist.  Fixing that bug revealed
    248 // that the expected results weren't correct anyway.
    249 // http://crbug.com/21321
    250 TEST_F(DomCheckerTest, DISABLED_Http) {
    251   if (!CommandLine::ForCurrentProcess()->HasSwitch(kRunDomCheckerTest))
    252     return;
    253 
    254   ResultsList new_passes, new_failures;
    255   RunTest(true, &new_passes, &new_failures);
    256   PrintResults(new_passes, new_failures);
    257 }
    258 
    259 }  // namespace
    260