1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/environment.h" 6 #include "base/file_util.h" 7 #include "base/path_service.h" 8 #include "base/process/launch.h" 9 #include "base/strings/string_split.h" 10 #include "base/strings/stringprintf.h" 11 #include "base/test/test_timeouts.h" 12 #include "base/time/time.h" 13 #include "chrome/browser/chrome_notification_types.h" 14 #include "chrome/browser/infobars/infobar.h" 15 #include "chrome/browser/infobars/infobar_service.h" 16 #include "chrome/browser/media/media_stream_infobar_delegate.h" 17 #include "chrome/browser/media/webrtc_browsertest_base.h" 18 #include "chrome/browser/media/webrtc_browsertest_common.h" 19 #include "chrome/browser/profiles/profile.h" 20 #include "chrome/browser/ui/browser.h" 21 #include "chrome/browser/ui/browser_tabstrip.h" 22 #include "chrome/browser/ui/tabs/tab_strip_model.h" 23 #include "chrome/common/chrome_switches.h" 24 #include "chrome/test/base/in_process_browser_test.h" 25 #include "chrome/test/base/ui_test_utils.h" 26 #include "chrome/test/perf/perf_test.h" 27 #include "chrome/test/ui/ui_test.h" 28 #include "content/public/browser/notification_service.h" 29 #include "content/public/test/browser_test_utils.h" 30 #include "net/test/python_utils.h" 31 #include "net/test/spawned_test_server/spawned_test_server.h" 32 33 static const base::FilePath::CharType kFrameAnalyzerExecutable[] = 34 #if defined(OS_WIN) 35 FILE_PATH_LITERAL("frame_analyzer.exe"); 36 #else 37 FILE_PATH_LITERAL("frame_analyzer"); 38 #endif 39 40 static const base::FilePath::CharType kArgbToI420ConverterExecutable[] = 41 #if defined(OS_WIN) 42 FILE_PATH_LITERAL("rgba_to_i420_converter.exe"); 43 #else 44 FILE_PATH_LITERAL("rgba_to_i420_converter"); 45 #endif 46 47 static const char kHomeEnvName[] = 48 #if defined(OS_WIN) 49 "HOMEPATH"; 50 #else 51 "HOME"; 52 #endif 53 54 // The working dir should be in the user's home folder. 55 static const base::FilePath::CharType kWorkingDirName[] = 56 FILE_PATH_LITERAL("webrtc_video_quality"); 57 static const base::FilePath::CharType kReferenceYuvFileName[] = 58 FILE_PATH_LITERAL("reference_video.yuv"); 59 static const base::FilePath::CharType kCapturedYuvFileName[] = 60 FILE_PATH_LITERAL("captured_video.yuv"); 61 static const base::FilePath::CharType kStatsFileName[] = 62 FILE_PATH_LITERAL("stats.txt"); 63 static const char kMainWebrtcTestHtmlPage[] = 64 "files/webrtc/webrtc_jsep01_test.html"; 65 static const char kCapturingWebrtcHtmlPage[] = 66 "files/webrtc/webrtc_video_quality_test.html"; 67 static const int kVgaWidth = 640; 68 static const int kVgaHeight = 480; 69 70 // If you change the port number, don't forget to modify video_extraction.js 71 // too! 72 static const char kPyWebSocketPortNumber[] = "12221"; 73 74 // Test the video quality of the WebRTC output. 75 // 76 // Prerequisites: This test case must run on a machine with a virtual webcam 77 // that plays video from the reference file located in <the running user's home 78 // folder>/kWorkingDirName/kReferenceYuvFileName. 79 // 80 // You must also compile the chromium_builder_webrtc target before you run this 81 // test to get all the tools built. 82 // 83 // The external compare_videos.py script also depends on two external 84 // executables which must be located in the PATH when running this test. 85 // * zxing (see the CPP version at https://code.google.com/p/zxing) 86 // * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org) 87 // 88 // The test case will launch a custom binary (peerconnection_server) which will 89 // allow two WebRTC clients to find each other. 90 // 91 // The test also runs several other custom binaries - rgba_to_i420 converter and 92 // frame_analyzer. Both tools can be found under third_party/webrtc/tools. The 93 // test also runs a stand alone Python implementation of a WebSocket server 94 // (pywebsocket) and a barcode_decoder script. 95 class WebrtcVideoQualityBrowserTest : public WebRtcTestBase { 96 public: 97 WebrtcVideoQualityBrowserTest() 98 : pywebsocket_server_(0), 99 environment_(base::Environment::Create()) {} 100 101 virtual void SetUpInProcessBrowserTestFixture() OVERRIDE { 102 peerconnection_server_.Start(); 103 104 // Ensure we have the stuff we need. 105 EXPECT_TRUE(base::PathExists(GetWorkingDir())) 106 << "Cannot find the working directory for the reference video and " 107 "the temporary files:" << GetWorkingDir().value(); 108 base::FilePath reference_file = 109 GetWorkingDir().Append(kReferenceYuvFileName); 110 EXPECT_TRUE(base::PathExists(reference_file)) 111 << "Cannot find the reference file to be used for video quality " 112 << "comparison: " << reference_file.value(); 113 } 114 115 virtual void TearDownInProcessBrowserTestFixture() OVERRIDE { 116 peerconnection_server_.Stop(); 117 ShutdownPyWebSocketServer(); 118 } 119 120 virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE { 121 // TODO(phoglund): check that user actually has the requisite devices and 122 // print a nice message if not; otherwise the test just times out which can 123 // be confusing. 124 // This test expects real device handling and requires a real webcam / audio 125 // device; it will not work with fake devices. 126 EXPECT_FALSE( 127 command_line->HasSwitch(switches::kUseFakeDeviceForMediaStream)) 128 << "You cannot run this test with fake devices."; 129 } 130 131 void StartPyWebSocketServer() { 132 base::FilePath path_pywebsocket_dir = 133 GetSourceDir().Append(FILE_PATH_LITERAL("third_party/pywebsocket/src")); 134 base::FilePath pywebsocket_server = path_pywebsocket_dir.Append( 135 FILE_PATH_LITERAL("mod_pywebsocket/standalone.py")); 136 base::FilePath path_to_data_handler = 137 GetSourceDir().Append(FILE_PATH_LITERAL("chrome/test/functional")); 138 139 EXPECT_TRUE(base::PathExists(pywebsocket_server)) 140 << "Fatal: missing pywebsocket server."; 141 EXPECT_TRUE(base::PathExists(path_to_data_handler)) 142 << "Fatal: missing data handler for pywebsocket server."; 143 144 AppendToPythonPath(path_pywebsocket_dir); 145 CommandLine pywebsocket_command = MakePythonCommand(pywebsocket_server); 146 147 // Construct the command line manually, the server doesn't support -arg=val. 148 pywebsocket_command.AppendArg("-p"); 149 pywebsocket_command.AppendArg(kPyWebSocketPortNumber); 150 pywebsocket_command.AppendArg("-d"); 151 pywebsocket_command.AppendArgPath(path_to_data_handler); 152 153 LOG(INFO) << "Running " << pywebsocket_command.GetCommandLineString(); 154 EXPECT_TRUE(base::LaunchProcess( 155 pywebsocket_command, base::LaunchOptions(), &pywebsocket_server_)) 156 << "Failed to launch pywebsocket server."; 157 } 158 159 void ShutdownPyWebSocketServer() { 160 EXPECT_TRUE(base::KillProcess(pywebsocket_server_, 0, false)) 161 << "Failed to shut down pywebsocket server!"; 162 } 163 164 // Convenience method which executes the provided javascript in the context 165 // of the provided web contents and returns what it evaluated to. 166 std::string ExecuteJavascript(const std::string& javascript, 167 content::WebContents* tab_contents) { 168 std::string result; 169 EXPECT_TRUE(content::ExecuteScriptAndExtractString( 170 tab_contents, javascript, &result)); 171 return result; 172 } 173 174 // Ensures we didn't get any errors asynchronously (e.g. while no javascript 175 // call from this test was outstanding). 176 // TODO(phoglund): this becomes obsolete when we switch to communicating with 177 // the DOM message queue. 178 void AssertNoAsynchronousErrors(content::WebContents* tab_contents) { 179 EXPECT_EQ("ok-no-errors", 180 ExecuteJavascript("getAnyTestFailures()", tab_contents)); 181 } 182 183 // The peer connection server lets our two tabs find each other and talk to 184 // each other (e.g. it is the application-specific "signaling solution"). 185 void ConnectToPeerConnectionServer(const std::string peer_name, 186 content::WebContents* tab_contents) { 187 std::string javascript = base::StringPrintf( 188 "connect('http://localhost:8888', '%s');", peer_name.c_str()); 189 EXPECT_EQ("ok-connected", ExecuteJavascript(javascript, tab_contents)); 190 } 191 192 void EstablishCall(content::WebContents* from_tab, 193 content::WebContents* to_tab) { 194 EXPECT_EQ("ok-peerconnection-created", 195 ExecuteJavascript("preparePeerConnection()", from_tab)); 196 EXPECT_EQ("ok-added", ExecuteJavascript("addLocalStream()", from_tab)); 197 EXPECT_EQ("ok-negotiating", ExecuteJavascript("negotiateCall()", from_tab)); 198 199 // Ensure the call gets up on both sides. 200 EXPECT_TRUE(PollingWaitUntil( 201 "getPeerConnectionReadyState()", "active", from_tab)); 202 EXPECT_TRUE(PollingWaitUntil( 203 "getPeerConnectionReadyState()", "active", to_tab)); 204 } 205 206 void HangUp(content::WebContents* from_tab) { 207 EXPECT_EQ("ok-call-hung-up", ExecuteJavascript("hangUp()", from_tab)); 208 } 209 210 void WaitUntilHangupVerified(content::WebContents* tab_contents) { 211 EXPECT_TRUE(PollingWaitUntil( 212 "getPeerConnectionReadyState()", "no-peer-connection", tab_contents)); 213 } 214 215 // Runs the RGBA to I420 converter on the video in |capture_video_filename|, 216 // which should contain frames of size |width| x |height|. 217 // 218 // The rgba_to_i420_converter is part of the webrtc_test_tools target which 219 // should be build prior to running this test. The resulting binary should 220 // live next to Chrome. 221 void RunARGBtoI420Converter(int width, 222 int height, 223 const base::FilePath& captured_video_filename) { 224 base::FilePath path_to_converter = base::MakeAbsoluteFilePath( 225 GetBrowserDir().Append(kArgbToI420ConverterExecutable)); 226 EXPECT_TRUE(base::PathExists(path_to_converter)) 227 << "Missing ARGB->I420 converter: should be in " 228 << path_to_converter.value(); 229 230 CommandLine converter_command(path_to_converter); 231 converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir()); 232 converter_command.AppendSwitchPath("--output_file", 233 captured_video_filename); 234 converter_command.AppendSwitchASCII("--width", 235 base::StringPrintf("%d", width)); 236 converter_command.AppendSwitchASCII("--height", 237 base::StringPrintf("%d", height)); 238 239 // We produce an output file that will later be used as an input to the 240 // barcode decoder and frame analyzer tools. 241 LOG(INFO) << "Running " << converter_command.GetCommandLineString(); 242 std::string result; 243 EXPECT_TRUE(base::GetAppOutput(converter_command, &result)); 244 LOG(INFO) << "Output was:\n\n" << result; 245 } 246 247 // Compares the |captured_video_filename| with the |reference_video_filename|. 248 // 249 // The barcode decoder decodes the captured video containing barcodes overlaid 250 // into every frame of the video (produced by rgba_to_i420_converter). It 251 // produces a set of PNG images and a |stats_file| that maps each captured 252 // frame to a frame in the reference video. The frames should be of size 253 // |width| x |height|. The output of compare_videos.py is returned. 254 std::string CompareVideos(int width, 255 int height, 256 const base::FilePath& captured_video_filename, 257 const base::FilePath& reference_video_filename, 258 const base::FilePath& stats_file) { 259 base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath( 260 GetBrowserDir().Append(kFrameAnalyzerExecutable)); 261 base::FilePath path_to_compare_script = GetSourceDir().Append( 262 FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py")); 263 264 EXPECT_TRUE(base::PathExists(path_to_analyzer)) 265 << "Missing frame analyzer: should be in " << path_to_analyzer.value(); 266 EXPECT_TRUE(base::PathExists(path_to_compare_script)) 267 << "Missing video compare script: should be in " 268 << path_to_compare_script.value(); 269 270 CommandLine compare_command = MakePythonCommand(path_to_compare_script); 271 compare_command.AppendSwitchPath("--ref_video", reference_video_filename); 272 compare_command.AppendSwitchPath("--test_video", captured_video_filename); 273 compare_command.AppendSwitchPath("--frame_analyzer", path_to_analyzer); 274 compare_command.AppendSwitchASCII("--yuv_frame_width", 275 base::StringPrintf("%d", width)); 276 compare_command.AppendSwitchASCII("--yuv_frame_height", 277 base::StringPrintf("%d", height)); 278 compare_command.AppendSwitchPath("--stats_file", stats_file); 279 280 LOG(INFO) << "Running " << compare_command.GetCommandLineString(); 281 std::string result; 282 EXPECT_TRUE(base::GetAppOutput(compare_command, &result)); 283 LOG(INFO) << "Output was:\n\n" << result; 284 return result; 285 } 286 287 // Processes the |frame_analyzer_output| for the different frame counts. 288 // 289 // The frame analyzer outputs additional information about the number of 290 // unique frames captured, The max number of repeated frames in a sequence and 291 // the max number of skipped frames. These values are then written to the Perf 292 // Graph. (Note: Some of the repeated or skipped frames will probably be due 293 // to the imperfection of JavaScript timers). 294 void PrintFramesCountPerfResults(std::string frame_analyzer_output) { 295 size_t unique_frames_pos = 296 frame_analyzer_output.rfind("Unique_frames_count"); 297 EXPECT_NE(unique_frames_pos, std::string::npos) 298 << "Missing Unique_frames_count in frame analyzer output:\n" 299 << frame_analyzer_output; 300 301 std::string unique_frame_counts = 302 frame_analyzer_output.substr(unique_frames_pos); 303 // TODO(phoglund): Fix ESTATS result to not have this silly newline. 304 std::replace( 305 unique_frame_counts.begin(), unique_frame_counts.end(), '\n', ' '); 306 307 std::vector<std::pair<std::string, std::string> > key_values; 308 base::SplitStringIntoKeyValuePairs( 309 unique_frame_counts, ':', ' ', &key_values); 310 std::vector<std::pair<std::string, std::string> >::const_iterator iter; 311 for (iter = key_values.begin(); iter != key_values.end(); ++iter) { 312 const std::pair<std::string, std::string>& key_value = *iter; 313 perf_test::PrintResult( 314 key_value.first, "", "VGA", key_value.second, "", false); 315 } 316 } 317 318 // Processes the |frame_analyzer_output| to extract the PSNR and SSIM values. 319 // 320 // The frame analyzer produces PSNR and SSIM results for every unique frame 321 // that has been captured. This method forms a list of all the psnr and ssim 322 // values and passes it to PrintResultList() for printing on the Perf Graph. 323 void PrintPsnrAndSsimPerfResults(std::string frame_analyzer_output) { 324 size_t stats_start = frame_analyzer_output.find("BSTATS"); 325 EXPECT_NE(stats_start, std::string::npos) 326 << "Missing BSTATS in frame analyzer output:\n" 327 << frame_analyzer_output; 328 size_t stats_end = frame_analyzer_output.find("ESTATS"); 329 EXPECT_NE(stats_end, std::string::npos) 330 << "Missing ESTATS in frame analyzer output:\n" 331 << frame_analyzer_output; 332 333 stats_start += std::string("BSTATS").size(); 334 std::string psnr_ssim_stats = 335 frame_analyzer_output.substr(stats_start, stats_end - stats_start); 336 337 // PSNR and SSIM values aren't really key-value pairs but it is convenient 338 // to parse them as such. 339 // TODO(phoglund): make the format more convenient so we need less 340 // processing here. 341 std::vector<std::pair<std::string, std::string> > psnr_ssim_entries; 342 base::SplitStringIntoKeyValuePairs( 343 psnr_ssim_stats, ' ', ';', &psnr_ssim_entries); 344 345 std::string psnr_value_list; 346 std::string ssim_value_list; 347 std::vector<std::pair<std::string, std::string> >::const_iterator iter; 348 for (iter = psnr_ssim_entries.begin(); iter != psnr_ssim_entries.end(); 349 ++iter) { 350 const std::pair<std::string, std::string>& psnr_and_ssim = *iter; 351 psnr_value_list.append(psnr_and_ssim.first).append(","); 352 ssim_value_list.append(psnr_and_ssim.second).append(","); 353 } 354 // Nuke last comma. 355 psnr_value_list.erase(psnr_value_list.size() - 1); 356 ssim_value_list.erase(ssim_value_list.size() - 1); 357 358 perf_test::PrintResultList("PSNR", "", "VGA", psnr_value_list, "dB", false); 359 perf_test::PrintResultList("SSIM", "", "VGA", ssim_value_list, "", false); 360 } 361 362 base::FilePath GetWorkingDir() { 363 std::string home_dir; 364 environment_->GetVar(kHomeEnvName, &home_dir); 365 base::FilePath::StringType native_home_dir(home_dir.begin(), 366 home_dir.end()); 367 return base::FilePath(native_home_dir).Append(kWorkingDirName); 368 } 369 370 private: 371 base::FilePath GetSourceDir() { 372 base::FilePath source_dir; 373 PathService::Get(base::DIR_SOURCE_ROOT, &source_dir); 374 return source_dir; 375 } 376 377 base::FilePath GetBrowserDir() { 378 base::FilePath browser_dir; 379 EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir)); 380 return browser_dir; 381 } 382 383 CommandLine MakePythonCommand(base::FilePath python_script) { 384 CommandLine python_command(CommandLine::NO_PROGRAM); 385 EXPECT_TRUE(GetPythonCommand(&python_command)); 386 CommandLine complete_command(python_script); 387 complete_command.PrependWrapper(python_command.GetCommandLineString()); 388 return complete_command; 389 } 390 391 PeerConnectionServerRunner peerconnection_server_; 392 base::ProcessHandle pywebsocket_server_; 393 scoped_ptr<base::Environment> environment_; 394 }; 395 396 #if defined(OS_WIN) 397 // Broken on Win: failing to start pywebsocket_server. http://crbug.com/255499. 398 #define MAYBE_MANUAL_TestVGAVideoQuality DISABLED_MANUAL_TestVGAVideoQuality 399 #else 400 #define MAYBE_MANUAL_TestVGAVideoQuality MANUAL_TestVGAVideoQuality 401 #endif 402 403 IN_PROC_BROWSER_TEST_F(WebrtcVideoQualityBrowserTest, 404 MAYBE_MANUAL_TestVGAVideoQuality) { 405 StartPyWebSocketServer(); 406 407 EXPECT_TRUE(test_server()->Start()); 408 409 ui_test_utils::NavigateToURL(browser(), 410 test_server()->GetURL(kMainWebrtcTestHtmlPage)); 411 content::WebContents* left_tab = 412 browser()->tab_strip_model()->GetActiveWebContents(); 413 GetUserMediaAndAccept(left_tab); 414 415 chrome::AddBlankTabAt(browser(), -1, true); 416 content::WebContents* right_tab = 417 browser()->tab_strip_model()->GetActiveWebContents(); 418 ui_test_utils::NavigateToURL(browser(), 419 test_server()->GetURL(kCapturingWebrtcHtmlPage)); 420 GetUserMediaAndAccept(right_tab); 421 422 ConnectToPeerConnectionServer("peer 1", left_tab); 423 ConnectToPeerConnectionServer("peer 2", right_tab); 424 425 EstablishCall(left_tab, right_tab); 426 427 AssertNoAsynchronousErrors(left_tab); 428 AssertNoAsynchronousErrors(right_tab); 429 430 // Poll slower here to avoid flooding the log with messages: capturing and 431 // sending frames take quite a bit of time. 432 int polling_interval_msec = 1000; 433 434 EXPECT_TRUE(PollingWaitUntil( 435 "doneFrameCapturing()", "done-capturing", right_tab, 436 polling_interval_msec)); 437 438 HangUp(left_tab); 439 WaitUntilHangupVerified(left_tab); 440 WaitUntilHangupVerified(right_tab); 441 442 AssertNoAsynchronousErrors(left_tab); 443 AssertNoAsynchronousErrors(right_tab); 444 445 EXPECT_TRUE(PollingWaitUntil( 446 "haveMoreFramesToSend()", "no-more-frames", right_tab, 447 polling_interval_msec)); 448 449 RunARGBtoI420Converter( 450 kVgaWidth, kVgaHeight, GetWorkingDir().Append(kCapturedYuvFileName)); 451 std::string output = 452 CompareVideos(kVgaWidth, 453 kVgaHeight, 454 GetWorkingDir().Append(kCapturedYuvFileName), 455 GetWorkingDir().Append(kReferenceYuvFileName), 456 GetWorkingDir().Append(kStatsFileName)); 457 458 PrintFramesCountPerfResults(output); 459 PrintPsnrAndSsimPerfResults(output); 460 } 461
