1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/command_line.h" 6 #include "base/file_util.h" 7 #include "base/files/file_enumerator.h" 8 #include "base/files/file_path.h" 9 #include "base/path_service.h" 10 #include "base/strings/string_number_conversions.h" 11 #include "base/strings/string_util.h" 12 #include "base/strings/stringprintf.h" 13 #include "content/public/browser/render_view_host.h" 14 #include "content/public/browser/render_widget_host_view.h" 15 #include "content/public/browser/web_contents.h" 16 #include "content/public/common/content_paths.h" 17 #include "content/public/common/content_switches.h" 18 #include "content/public/test/browser_test_utils.h" 19 #include "content/shell/shell.h" 20 #include "content/test/content_browser_test.h" 21 #include "content/test/content_browser_test_utils.h" 22 #include "gpu/config/gpu_test_config.h" 23 #include "net/base/net_util.h" 24 #include "testing/gtest/include/gtest/gtest.h" 25 #include "third_party/skia/include/core/SkBitmap.h" 26 #include "third_party/skia/include/core/SkColor.h" 27 #include "ui/gfx/codec/png_codec.h" 28 #include "ui/gfx/size.h" 29 #include "ui/gl/gl_switches.h" 30 #include "ui/snapshot/snapshot.h" 31 32 namespace { 33 34 enum ReferenceImageOption { 35 kReferenceImageLocal, 36 kReferenceImageCheckedIn, 37 kReferenceImageNone // Only check a few key pixels. 38 }; 39 40 struct ReferencePixel { 41 int x, y; 42 unsigned char r, g, b; 43 }; 44 45 // Command line flag for overriding the default location for putting generated 46 // test images that do not match references. 47 const char kGeneratedDir[] = "generated-dir"; 48 // Command line flag for overriding the default location for reference images. 49 const char kReferenceDir[] = "reference-dir"; 50 // Command line flag for Chromium build revision. 51 const char kBuildRevision[] = "build-revision"; 52 53 // Reads and decodes a PNG image to a bitmap. Returns true on success. The PNG 54 // should have been encoded using |gfx::PNGCodec::Encode|. 55 bool ReadPNGFile(const base::FilePath& file_path, SkBitmap* bitmap) { 56 DCHECK(bitmap); 57 base::FilePath abs_path(base::MakeAbsoluteFilePath(file_path)); 58 if (abs_path.empty()) 59 return false; 60 61 std::string png_data; 62 return file_util::ReadFileToString(abs_path, &png_data) && 63 gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&png_data[0]), 64 png_data.length(), 65 bitmap); 66 } 67 68 // Encodes a bitmap into a PNG and write to disk. Returns true on success. The 69 // parent directory does not have to exist. 70 bool WritePNGFile(const SkBitmap& bitmap, const base::FilePath& file_path) { 71 std::vector<unsigned char> png_data; 72 if (gfx::PNGCodec::EncodeBGRASkBitmap(bitmap, true, &png_data) && 73 file_util::CreateDirectory(file_path.DirName())) { 74 int bytes_written = file_util::WriteFile( 75 file_path, reinterpret_cast<char*>(&png_data[0]), png_data.size()); 76 if (bytes_written == static_cast<int>(png_data.size())) 77 return true; 78 } 79 return false; 80 } 81 82 // Write an empty file, whose name indicates the chrome revision when the ref 83 // image was generated. 84 bool WriteREVFile(const base::FilePath& file_path) { 85 if (file_util::CreateDirectory(file_path.DirName())) { 86 char one_byte = 0; 87 int bytes_written = file_util::WriteFile(file_path, &one_byte, 1); 88 if (bytes_written == 1) 89 return true; 90 } 91 return false; 92 } 93 94 } // namespace anonymous 95 96 namespace content { 97 98 // Test fixture for GPU image comparison tests. 99 // TODO(kkania): Document how to add to/modify these tests. 100 class GpuPixelBrowserTest : public ContentBrowserTest { 101 public: 102 GpuPixelBrowserTest() 103 : ref_img_revision_(0), 104 ref_img_revision_no_older_than_(0), 105 ref_img_option_(kReferenceImageNone) { 106 } 107 108 virtual void SetUp() { 109 // We expect real pixel output for these tests. 110 UseRealGLContexts(); 111 112 ContentBrowserTest::SetUp(); 113 } 114 115 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 116 command_line->AppendSwitchASCII(switches::kTestGLLib, 117 "libllvmpipe.so"); 118 } 119 120 virtual void SetUpInProcessBrowserTestFixture() OVERRIDE { 121 ContentBrowserTest::SetUpInProcessBrowserTestFixture(); 122 123 CommandLine* command_line = CommandLine::ForCurrentProcess(); 124 if (command_line->HasSwitch(switches::kUseGpuInTests)) 125 ref_img_option_ = kReferenceImageLocal; 126 127 if (command_line->HasSwitch(kBuildRevision)) 128 build_revision_ = command_line->GetSwitchValueASCII(kBuildRevision); 129 130 ASSERT_TRUE(PathService::Get(DIR_TEST_DATA, &test_data_dir_)); 131 test_data_dir_ = test_data_dir_.AppendASCII("gpu"); 132 133 if (command_line->HasSwitch(kGeneratedDir)) 134 generated_img_dir_ = command_line->GetSwitchValuePath(kGeneratedDir); 135 else 136 generated_img_dir_ = test_data_dir_.AppendASCII("generated"); 137 138 switch (ref_img_option_) { 139 case kReferenceImageLocal: 140 if (command_line->HasSwitch(kReferenceDir)) 141 ref_img_dir_ = command_line->GetSwitchValuePath(kReferenceDir); 142 else 143 ref_img_dir_ = test_data_dir_.AppendASCII("gpu_reference"); 144 break; 145 case kReferenceImageCheckedIn: 146 ref_img_dir_ = test_data_dir_.AppendASCII("llvmpipe_reference"); 147 break; 148 default: 149 break; 150 } 151 152 test_name_ = testing::UnitTest::GetInstance()->current_test_info()->name(); 153 const char* test_status_prefixes[] = { 154 "DISABLED_", "FLAKY_", "FAILS_", "MANUAL_"}; 155 for (size_t i = 0; i < arraysize(test_status_prefixes); ++i) { 156 ReplaceFirstSubstringAfterOffset( 157 &test_name_, 0, test_status_prefixes[i], std::string()); 158 } 159 } 160 161 // If the existing ref image was saved from an revision older than the 162 // ref_img_update_revision, refresh the ref image. 163 void RunPixelTest(const gfx::Size& tab_container_size, 164 const base::FilePath& url, 165 int64 ref_img_update_revision, 166 const ReferencePixel* ref_pixels, 167 size_t ref_pixel_count) { 168 if (ref_img_option_ == kReferenceImageLocal) { 169 ref_img_revision_no_older_than_ = ref_img_update_revision; 170 ObtainLocalRefImageRevision(); 171 } 172 173 DOMMessageQueue message_queue; 174 NavigateToURL(shell(), net::FilePathToFileURL(url)); 175 176 std::string message; 177 // Wait for notification that page is loaded. 178 ASSERT_TRUE(message_queue.WaitForMessage(&message)); 179 EXPECT_STREQ("\"SUCCESS\"", message.c_str()) << message; 180 181 SkBitmap bitmap; 182 ASSERT_TRUE(TabSnapShotToImage(&bitmap, tab_container_size)); 183 bool same_pixels = true; 184 if (ref_img_option_ == kReferenceImageNone && ref_pixels && ref_pixel_count) 185 same_pixels = ComparePixels(bitmap, ref_pixels, ref_pixel_count); 186 else 187 same_pixels = CompareImages(bitmap); 188 EXPECT_TRUE(same_pixels); 189 } 190 191 const base::FilePath& test_data_dir() const { 192 return test_data_dir_; 193 } 194 195 private: 196 base::FilePath test_data_dir_; 197 base::FilePath generated_img_dir_; 198 base::FilePath ref_img_dir_; 199 int64 ref_img_revision_; 200 std::string build_revision_; 201 // The name of the test, with any special prefixes dropped. 202 std::string test_name_; 203 204 // Any local ref image generated from older revision is ignored. 205 int64 ref_img_revision_no_older_than_; 206 207 // Whether use locally generated ref images, or checked in ref images, or 208 // simply check a few key pixels. 209 ReferenceImageOption ref_img_option_; 210 211 // Compares the generated bitmap with the appropriate reference image on disk. 212 // Returns true iff the images were the same. 213 // 214 // If no valid reference image exists, save the generated bitmap to the disk. 215 // The image format is: 216 // <test_name>_<revision>.png 217 // E.g., 218 // WebGLTeapot_19762.png 219 // The number is the chromium revision that generated the image. 220 // 221 // On failure or on ref image generation, the image and diff image will be 222 // written to disk. The formats are: 223 // FAIL_<ref_image_name>, DIFF_<ref_image_name> 224 // E.g., 225 // FAIL_WebGLTeapot_19762.png, DIFF_WebGLTeapot_19762.png 226 bool CompareImages(const SkBitmap& gen_bmp) { 227 SkBitmap ref_bmp_on_disk; 228 229 base::FilePath img_path = ref_img_dir_.AppendASCII(test_name_ + ".png"); 230 bool found_ref_img = ReadPNGFile(img_path, &ref_bmp_on_disk); 231 232 if (!found_ref_img && ref_img_option_ == kReferenceImageCheckedIn) { 233 LOG(ERROR) << "Couldn't find reference image: " 234 << img_path.value(); 235 // No image to compare to, exit early. 236 return false; 237 } 238 239 const SkBitmap* ref_bmp; 240 bool save_gen = false; 241 bool save_diff = true; 242 bool rt = true; 243 244 if ((ref_img_revision_ <= 0 && ref_img_option_ == kReferenceImageLocal) || 245 !found_ref_img) { 246 base::FilePath rev_path = ref_img_dir_.AppendASCII( 247 test_name_ + "_" + build_revision_ + ".rev"); 248 if (!WritePNGFile(gen_bmp, img_path)) { 249 LOG(ERROR) << "Can't save generated image to: " 250 << img_path.value() 251 << " as future reference."; 252 rt = false; 253 } else { 254 LOG(INFO) << "Saved reference image to: " 255 << img_path.value(); 256 } 257 if (rt) { 258 if (!WriteREVFile(rev_path)) { 259 LOG(ERROR) << "Can't save revision file to: " 260 << rev_path.value(); 261 rt = false; 262 base::DeleteFile(img_path, false); 263 } else { 264 LOG(INFO) << "Saved revision file to: " 265 << rev_path.value(); 266 } 267 } 268 if (ref_img_revision_ > 0) { 269 LOG(ERROR) << "Can't read the local ref image: " 270 << img_path.value() 271 << ", reset it."; 272 rt = false; 273 } 274 // If we re-generate the ref image, we save the gen and diff images so 275 // the ref image can be uploaded to the server and be viewed later. 276 save_gen = true; 277 save_diff = true; 278 ref_bmp = &gen_bmp; 279 } else { 280 ref_bmp = &ref_bmp_on_disk; 281 } 282 283 SkBitmap diff_bmp; 284 if (ref_bmp->width() != gen_bmp.width() || 285 ref_bmp->height() != gen_bmp.height()) { 286 LOG(ERROR) 287 << "Dimensions do not match (Expected) vs (Actual):" 288 << "(" << ref_bmp->width() << "x" << ref_bmp->height() 289 << ") vs. " 290 << "(" << gen_bmp.width() << "x" << gen_bmp.height() << ")"; 291 if (ref_img_option_ == kReferenceImageLocal) 292 save_gen = true; 293 rt = false; 294 } else { 295 // Compare pixels and create a simple diff image. 296 int diff_pixels_count = 0; 297 diff_bmp.setConfig(SkBitmap::kARGB_8888_Config, 298 gen_bmp.width(), gen_bmp.height()); 299 diff_bmp.allocPixels(); 300 diff_bmp.eraseColor(SK_ColorWHITE); 301 SkAutoLockPixels lock_bmp(gen_bmp); 302 SkAutoLockPixels lock_ref_bmp(*ref_bmp); 303 SkAutoLockPixels lock_diff_bmp(diff_bmp); 304 // The reference images were saved with no alpha channel. Use the mask to 305 // set alpha to 0. 306 uint32_t kAlphaMask = 0x00FFFFFF; 307 for (int x = 0; x < gen_bmp.width(); ++x) { 308 for (int y = 0; y < gen_bmp.height(); ++y) { 309 if ((*gen_bmp.getAddr32(x, y) & kAlphaMask) != 310 (*ref_bmp->getAddr32(x, y) & kAlphaMask)) { 311 ++diff_pixels_count; 312 *diff_bmp.getAddr32(x, y) = 192 << 16; // red 313 } 314 } 315 } 316 if (diff_pixels_count > 0) { 317 LOG(ERROR) << diff_pixels_count 318 << " pixels do not match."; 319 if (ref_img_option_ == kReferenceImageLocal) { 320 save_gen = true; 321 save_diff = true; 322 } 323 rt = false; 324 } 325 } 326 327 std::string ref_img_filename = img_path.BaseName().MaybeAsASCII(); 328 if (save_gen) { 329 base::FilePath img_fail_path = generated_img_dir_.AppendASCII( 330 "FAIL_" + ref_img_filename); 331 if (!WritePNGFile(gen_bmp, img_fail_path)) { 332 LOG(ERROR) << "Can't save generated image to: " 333 << img_fail_path.value(); 334 } else { 335 LOG(INFO) << "Saved generated image to: " 336 << img_fail_path.value(); 337 } 338 } 339 if (save_diff) { 340 base::FilePath img_diff_path = generated_img_dir_.AppendASCII( 341 "DIFF_" + ref_img_filename); 342 if (!WritePNGFile(diff_bmp, img_diff_path)) { 343 LOG(ERROR) << "Can't save generated diff image to: " 344 << img_diff_path.value(); 345 } else { 346 LOG(INFO) << "Saved difference image to: " 347 << img_diff_path.value(); 348 } 349 } 350 return rt; 351 } 352 353 bool ComparePixels(const SkBitmap& gen_bmp, 354 const ReferencePixel* ref_pixels, 355 size_t ref_pixel_count) { 356 SkAutoLockPixels lock_bmp(gen_bmp); 357 358 for (size_t i = 0; i < ref_pixel_count; ++i) { 359 int x = ref_pixels[i].x; 360 int y = ref_pixels[i].y; 361 unsigned char r = ref_pixels[i].r; 362 unsigned char g = ref_pixels[i].g; 363 unsigned char b = ref_pixels[i].b; 364 365 DCHECK(x >= 0 && x < gen_bmp.width() && y >= 0 && y < gen_bmp.height()); 366 367 unsigned char* rgba = reinterpret_cast<unsigned char*>( 368 gen_bmp.getAddr32(x, y)); 369 DCHECK(rgba); 370 if (rgba[0] != b || rgba[1] != g || rgba[2] != r) { 371 std::string error_message = base::StringPrintf( 372 "pixel(%d,%d) expects [%u,%u,%u], but gets [%u,%u,%u] instead", 373 x, y, r, g, b, rgba[0], rgba[1], rgba[2]); 374 LOG(ERROR) << error_message.c_str(); 375 return false; 376 } 377 } 378 return true; 379 } 380 381 // Take snapshot of the tab, encode it as PNG, and save to a SkBitmap. 382 bool TabSnapShotToImage(SkBitmap* bitmap, const gfx::Size& size) { 383 CHECK(bitmap); 384 std::vector<unsigned char> png; 385 386 gfx::Rect snapshot_bounds(size); 387 RenderViewHost* view_host = shell()->web_contents()->GetRenderViewHost(); 388 if (!ui::GrabViewSnapshot(view_host->GetView()->GetNativeView(), 389 &png, snapshot_bounds)) { 390 LOG(ERROR) << "ui::GrabViewSnapShot() failed"; 391 return false; 392 } 393 394 if (!gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&*png.begin()), 395 png.size(), bitmap)) { 396 LOG(ERROR) << "Decode PNG to a SkBitmap failed"; 397 return false; 398 } 399 return true; 400 } 401 402 // If no valid local revision file is located, the ref_img_revision_ is 0. 403 void ObtainLocalRefImageRevision() { 404 base::FilePath filter; 405 filter = filter.AppendASCII(test_name_ + "_*.rev"); 406 base::FileEnumerator locator(ref_img_dir_, 407 false, // non recursive 408 base::FileEnumerator::FILES, 409 filter.value()); 410 int64 max_revision = 0; 411 std::vector<base::FilePath> outdated_revs; 412 for (base::FilePath full_path = locator.Next(); 413 !full_path.empty(); 414 full_path = locator.Next()) { 415 std::string filename = 416 full_path.BaseName().RemoveExtension().MaybeAsASCII(); 417 std::string revision_string = 418 filename.substr(test_name_.length() + 1); 419 int64 revision = 0; 420 bool converted = base::StringToInt64(revision_string, &revision); 421 if (!converted) 422 continue; 423 if (revision < ref_img_revision_no_older_than_ || 424 revision < max_revision) { 425 outdated_revs.push_back(full_path); 426 continue; 427 } 428 max_revision = revision; 429 } 430 ref_img_revision_ = max_revision; 431 for (size_t i = 0; i < outdated_revs.size(); ++i) 432 base::DeleteFile(outdated_revs[i], false); 433 } 434 435 DISALLOW_COPY_AND_ASSIGN(GpuPixelBrowserTest); 436 }; 437 438 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest, MANUAL_WebGLGreenTriangle) { 439 // If test baseline needs to be updated after a given revision, update the 440 // following number. If no revision requirement, then 0. 441 const int64 ref_img_revision_update = 123489; 442 443 const ReferencePixel ref_pixels[] = { 444 // x, y, r, g, b 445 {50, 100, 0, 0, 0}, 446 {100, 100, 0, 255, 0}, 447 {150, 100, 0, 0, 0}, 448 {50, 150, 0, 255, 0}, 449 {100, 150, 0, 255, 0}, 450 {150, 150, 0, 255, 0} 451 }; 452 const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel); 453 454 gfx::Size container_size(400, 300); 455 base::FilePath url = 456 test_data_dir().AppendASCII("pixel_webgl.html"); 457 RunPixelTest(container_size, url, ref_img_revision_update, 458 ref_pixels, ref_pixel_count); 459 } 460 461 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest, MANUAL_CSS3DBlueBox) { 462 // If test baseline needs to be updated after a given revision, update the 463 // following number. If no revision requirement, then 0. 464 const int64 ref_img_revision_update = 209827; 465 466 const ReferencePixel ref_pixels[] = { 467 // x, y, r, g, b 468 {70, 50, 0, 0, 255}, 469 {150, 50, 0, 0, 0}, 470 {70, 90, 0, 0, 255}, 471 {150, 90, 0, 0, 255}, 472 {70, 125, 0, 0, 255}, 473 {150, 125, 0, 0, 0} 474 }; 475 const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel); 476 477 gfx::Size container_size(400, 300); 478 base::FilePath url = 479 test_data_dir().AppendASCII("pixel_css3d.html"); 480 RunPixelTest(container_size, url, ref_img_revision_update, 481 ref_pixels, ref_pixel_count); 482 } 483 484 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest, MANUAL_Canvas2DRedBoxHD) { 485 // If test baseline needs to be updated after a given revision, update the 486 // following number. If no revision requirement, then 0. 487 const int64 ref_img_revision_update = 123489; 488 489 const ReferencePixel ref_pixels[] = { 490 // x, y, r, g, b 491 {40, 100, 0, 0, 0}, 492 {60, 100, 127, 0, 0}, 493 {140, 100, 127, 0, 0}, 494 {160, 100, 0, 0, 0} 495 }; 496 const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel); 497 498 gfx::Size container_size(400, 300); 499 base::FilePath url = 500 test_data_dir().AppendASCII("pixel_canvas2d.html"); 501 RunPixelTest(container_size, url, ref_img_revision_update, 502 ref_pixels, ref_pixel_count); 503 } 504 505 class GpuPixelTestCanvas2DSD : public GpuPixelBrowserTest { 506 public: 507 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 508 GpuPixelBrowserTest::SetUpCommandLine(command_line); 509 command_line->AppendSwitch(switches::kDisableAccelerated2dCanvas); 510 } 511 }; 512 513 IN_PROC_BROWSER_TEST_F(GpuPixelTestCanvas2DSD, MANUAL_Canvas2DRedBoxSD) { 514 // If test baseline needs to be updated after a given revision, update the 515 // following number. If no revision requirement, then 0. 516 const int64 ref_img_revision_update = 123489; 517 518 const ReferencePixel ref_pixels[] = { 519 // x, y, r, g, b 520 {40, 100, 0, 0, 0}, 521 {60, 100, 127, 0, 0}, 522 {140, 100, 127, 0, 0}, 523 {160, 100, 0, 0, 0} 524 }; 525 const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel); 526 527 gfx::Size container_size(400, 300); 528 base::FilePath url = 529 test_data_dir().AppendASCII("pixel_canvas2d.html"); 530 RunPixelTest(container_size, url, ref_img_revision_update, 531 ref_pixels, ref_pixel_count); 532 } 533 534 class GpuPixelTestBrowserPlugin : public GpuPixelBrowserTest { 535 public: 536 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 537 GpuPixelBrowserTest::SetUpCommandLine(command_line); 538 command_line->AppendSwitch(switches::kEnableBrowserPluginForAllViewTypes); 539 } 540 }; 541 542 // TODO(fsamuel): re-enable as MANUAL_BrowserPluginBlueBox: crbug.com/166165 543 IN_PROC_BROWSER_TEST_F(GpuPixelTestBrowserPlugin, 544 DISABLED_BrowserPluginBlueBox) { 545 // If test baseline needs to be updated after a given revision, update the 546 // following number. If no revision requirement, then 0. 547 const int64 ref_img_revision_update = 209445; 548 549 const ReferencePixel ref_pixels[] = { 550 // x, y, r, g, b 551 {70, 50, 0, 0, 255}, 552 {150, 50, 0, 0, 0}, 553 {70, 90, 0, 0, 255}, 554 {150, 90, 0, 0, 255}, 555 {70, 125, 0, 0, 255}, 556 {150, 125, 0, 0, 0} 557 }; 558 const size_t ref_pixel_count = sizeof(ref_pixels) / sizeof(ReferencePixel); 559 560 gfx::Size container_size(400, 300); 561 base::FilePath url = 562 test_data_dir().AppendASCII("pixel_browser_plugin.html"); 563 RunPixelTest(container_size, url, ref_img_revision_update, 564 ref_pixels, ref_pixel_count); 565 } 566 567 } // namespace content 568 569
