1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 // Implementation of Media Optimization Test 12 // testing is done via the VCM module, no specific Media opt functionality. 13 14 #include "webrtc/modules/video_coding/main/test/media_opt_test.h" 15 16 #include <stdio.h> 17 #include <string.h> 18 #include <time.h> 19 #include <vector> 20 21 #include "webrtc/modules/rtp_rtcp/interface/rtp_receiver.h" 22 #include "webrtc/modules/video_coding/main/interface/video_coding.h" 23 #include "webrtc/modules/video_coding/main/test/test_macros.h" 24 #include "webrtc/modules/video_coding/main/test/test_util.h" 25 #include "webrtc/test/testsupport/fileutils.h" 26 #include "webrtc/test/testsupport/metrics/video_metrics.h" 27 28 using namespace webrtc; 29 30 int MediaOptTest::RunTest(int testNum, CmdArgs& args) 31 { 32 Trace::CreateTrace(); 33 Trace::SetTraceFile((test::OutputPath() + "mediaOptTestTrace.txt").c_str()); 34 Trace::set_level_filter(webrtc::kTraceAll); 35 VideoCodingModule* vcm = VideoCodingModule::Create(); 36 Clock* clock = Clock::GetRealTimeClock(); 37 MediaOptTest* mot = new MediaOptTest(vcm, clock); 38 if (testNum == 0) 39 { // regular 40 mot->Setup(0, args); 41 mot->GeneralSetup(); 42 mot->Perform(); 43 mot->Print(1);// print to screen 44 mot->TearDown(); 45 } 46 if (testNum == 1) 47 { // release test 48 mot->Setup(0, args); 49 mot->RTTest(); 50 } 51 if (testNum == 2) 52 { // release test, running from script 53 mot->Setup(1, args); 54 mot->GeneralSetup(); 55 mot->Perform(); 56 mot->Print(1);// print to screen 57 mot->TearDown(); 58 } 59 60 VideoCodingModule::Destroy(vcm); 61 delete mot; 62 Trace::ReturnTrace(); 63 return 0; 64 65 } 66 67 68 MediaOptTest::MediaOptTest(VideoCodingModule* vcm, Clock* clock) 69 : _vcm(vcm), 70 _rtp(NULL), 71 _outgoingTransport(NULL), 72 _dataCallback(NULL), 73 _clock(clock), 74 _width(0), 75 _height(0), 76 _lengthSourceFrame(0), 77 _timeStamp(0), 78 _frameRate(30.0f), 79 _nackEnabled(false), 80 _fecEnabled(false), 81 _rttMS(0), 82 _bitRate(300.0f), 83 _lossRate(0.0f), 84 _renderDelayMs(0), 85 _frameCnt(0), 86 _sumEncBytes(0), 87 _numFramesDropped(0), 88 _numberOfCores(4) { 89 } 90 91 MediaOptTest::~MediaOptTest() { 92 delete _rtp; 93 } 94 95 void MediaOptTest::Setup(int testType, CmdArgs& args) { 96 /*TEST USER SETTINGS*/ 97 // test parameters 98 _inname = args.inputFile; 99 if (args.outputFile == "") 100 _outname = test::OutputPath() + "MOTest_out.vp8"; 101 else 102 _outname = args.outputFile; 103 // actual source after frame dropping 104 _actualSourcename = test::OutputPath() + "MOTestSource.yuv"; 105 _codecName = args.codecName; 106 _sendCodecType = args.codecType; 107 _width = args.width; 108 _height = args.height; 109 _frameRate = args.frameRate; 110 _bitRate = args.bitRate; 111 _numberOfCores = 4; 112 113 // error resilience 114 _nackEnabled = false; 115 _fecEnabled = true; 116 _nackFecEnabled = false; 117 118 _rttMS = 100; 119 _lossRate = 0.00*255; // no packet loss 120 121 _testType = testType; 122 123 //For multiple runs with script 124 if (_testType == 1) 125 { 126 float rateTest,lossTest; 127 int numRuns; 128 _fpinp = fopen("dat_inp","rb"); 129 _fpout = fopen("test_runs/dat_out","ab"); 130 _fpout2 = fopen("test_runs/dat_out2","ab"); 131 TEST(fscanf(_fpinp,"%f %f %d \n",&rateTest,&lossTest,&numRuns) > 0); 132 _bitRate = rateTest; 133 _lossRate = lossTest; 134 _testNum = 0; 135 136 // for bit rates: 500, 1000, 2000, 3000,4000 137 // for loss rates: 0, 1, 3, 5, 10% 138 _numParRuns = 25; 139 140 _testNum = numRuns + 1; 141 if (rateTest == 0.0) _lossRate = 0.0; 142 else 143 { 144 if (rateTest == 4000) //final bit rate 145 { 146 if (lossTest == 0.1*255) _lossRate = 0.0; //start at 1% 147 else 148 if (lossTest == 0.05*255) _lossRate = 0.1*255; //final loss rate 149 else 150 if (lossTest == 0.0) _lossRate = 0.01*255; 151 else _lossRate = lossTest + 0.02*255; 152 } 153 } 154 155 if (rateTest == 0.0 || rateTest == 4000) _bitRate = 500; //starting bit rate 156 else 157 if (rateTest == 500) _bitRate = 1000; 158 else _bitRate = rateTest + 1000; 159 } 160 // 161 162 _renderDelayMs = 0; 163 /* test settings end*/ 164 165 _lengthSourceFrame = 3*_width*_height/2; 166 _log.open((test::OutputPath() + "VCM_MediaOptLog.txt").c_str(), 167 std::fstream::out | std::fstream::app); 168 } 169 170 void 171 MediaOptTest::GeneralSetup() 172 { 173 uint32_t minPlayoutDelayMs = 0; 174 175 if ((_sourceFile = fopen(_inname.c_str(), "rb")) == NULL) 176 { 177 printf("Cannot read file %s.\n", _inname.c_str()); 178 exit(1); 179 } 180 181 if ((_decodedFile = fopen(_outname.c_str(), "wb")) == NULL) 182 { 183 printf("Cannot read file %s.\n", _outname.c_str()); 184 exit(1); 185 } 186 187 if ((_actualSourceFile = fopen(_actualSourcename.c_str(), "wb")) == NULL) 188 { 189 printf("Cannot read file %s.\n", _actualSourcename.c_str()); 190 exit(1); 191 } 192 if (_vcm->InitializeReceiver() < 0) 193 { 194 exit(1); 195 } 196 if (_vcm->InitializeSender()) 197 { 198 exit(1); 199 } 200 _outgoingTransport = new RTPSendCompleteCallback(_clock); 201 _dataCallback = new RtpDataCallback(_vcm); 202 203 RtpRtcp::Configuration configuration; 204 configuration.id = 1; 205 configuration.audio = false; 206 configuration.outgoing_transport = _outgoingTransport; 207 _rtp = RtpRtcp::CreateRtpRtcp(configuration); 208 209 _outgoingTransport->SetRtpModule(_rtp); 210 211 // Registering codecs for the RTP module 212 213 // Register receive and send payload 214 VideoCodec video_codec; 215 strncpy(video_codec.plName, "VP8", 32); 216 video_codec.plType = VCM_VP8_PAYLOAD_TYPE; 217 rtp_receiver_->RegisterReceivePayload(video_codec.plName, 218 video_codec.plType, 219 90000, 220 0, 221 video_codec.maxBitrate); 222 _rtp->RegisterSendPayload(video_codec); 223 224 strncpy(video_codec.plName, "ULPFEC", 32); 225 video_codec.plType = VCM_ULPFEC_PAYLOAD_TYPE; 226 rtp_receiver_->RegisterReceivePayload(video_codec.plName, 227 video_codec.plType, 228 90000, 229 0, 230 video_codec.maxBitrate); 231 _rtp->RegisterSendPayload(video_codec); 232 233 strncpy(video_codec.plName, "RED", 32); 234 video_codec.plType = VCM_RED_PAYLOAD_TYPE; 235 rtp_receiver_->RegisterReceivePayload(video_codec.plName, 236 video_codec.plType, 237 90000, 238 0, 239 video_codec.maxBitrate); 240 _rtp->RegisterSendPayload(video_codec); 241 242 if (_nackFecEnabled == 1) 243 _rtp->SetGenericFECStatus(_nackFecEnabled, VCM_RED_PAYLOAD_TYPE, 244 VCM_ULPFEC_PAYLOAD_TYPE); 245 else 246 _rtp->SetGenericFECStatus(_fecEnabled, VCM_RED_PAYLOAD_TYPE, 247 VCM_ULPFEC_PAYLOAD_TYPE); 248 249 // VCM: Registering codecs 250 VideoCodec sendCodec; 251 _vcm->InitializeSender(); 252 _vcm->InitializeReceiver(); 253 int32_t numberOfCodecs = _vcm->NumberOfCodecs(); 254 if (numberOfCodecs < 1) 255 { 256 exit(1); 257 } 258 259 if (_vcm->Codec(_sendCodecType, &sendCodec) != 0) 260 { 261 printf("Unknown codec\n"); 262 exit(1); 263 } 264 // register codec 265 sendCodec.startBitrate = (int) _bitRate; 266 sendCodec.height = _height; 267 sendCodec.width = _width; 268 sendCodec.maxFramerate = (uint8_t)_frameRate; 269 _vcm->RegisterSendCodec(&sendCodec, _numberOfCores, 1440); 270 _vcm->RegisterReceiveCodec(&sendCodec, _numberOfCores); // same settings for encode and decode 271 272 _vcm->SetRenderDelay(_renderDelayMs); 273 _vcm->SetMinimumPlayoutDelay(minPlayoutDelayMs); 274 } 275 // The following test shall be conducted under release tests 276 277 278 279 int32_t 280 MediaOptTest::Perform() 281 { 282 VCMDecodeCompleteCallback receiveCallback(_decodedFile); 283 284 VCMRTPEncodeCompleteCallback* encodeCompleteCallback = new VCMRTPEncodeCompleteCallback(_rtp); 285 _vcm->RegisterTransportCallback(encodeCompleteCallback); 286 encodeCompleteCallback->SetCodecType(ConvertCodecType(_codecName.c_str())); 287 encodeCompleteCallback->SetFrameDimensions(_width, _height); 288 289 // callback settings 290 VideoProtectionCallback protectionCallback; 291 protectionCallback.RegisterRtpModule(_rtp); 292 _vcm->RegisterProtectionCallback(&protectionCallback); 293 294 // set error resilience / test parameters: 295 _outgoingTransport->SetLossPct(_lossRate); 296 if (_nackFecEnabled == 1) { 297 _vcm->SetVideoProtection(kProtectionNackFEC, _nackFecEnabled); 298 } else { 299 _vcm->SetVideoProtection(kProtectionNack, _nackEnabled); 300 _vcm->SetVideoProtection(kProtectionFEC, _fecEnabled); 301 } 302 303 // START TEST 304 I420VideoFrame sourceFrame; 305 uint8_t* tmpBuffer = new uint8_t[_lengthSourceFrame]; 306 _vcm->SetChannelParameters(static_cast<uint32_t>(1000 * _bitRate), 307 (uint8_t)_lossRate, _rttMS); 308 _vcm->RegisterReceiveCallback(&receiveCallback); 309 310 _frameCnt = 0; 311 _sumEncBytes = 0.0; 312 _numFramesDropped = 0; 313 int half_width = (_width + 1) / 2; 314 int half_height = (_height + 1) / 2; 315 int size_y = _width * _height; 316 int size_uv = half_width * half_height; 317 318 while (feof(_sourceFile)== 0) 319 { 320 TEST(fread(tmpBuffer, 1, _lengthSourceFrame, _sourceFile) > 0); 321 _frameCnt++; 322 sourceFrame.CreateFrame(size_y, tmpBuffer, 323 size_uv, tmpBuffer + size_y, 324 size_uv, tmpBuffer + size_y + size_uv, 325 _width, _height, 326 _width, half_width, half_width); 327 _timeStamp += (uint32_t)(9e4 / static_cast<float>(_frameRate)); 328 sourceFrame.set_timestamp(_timeStamp); 329 TEST(_vcm->AddVideoFrame(sourceFrame) == VCM_OK); 330 // inform RTP Module of error resilience features 331 //_rtp->SetFECCodeRate(protectionCallback.FECKeyRate(),protectionCallback.FECDeltaRate()); 332 //_rtp->SetNACKStatus(protectionCallback.NACKMethod()); 333 334 int32_t ret = _vcm->Decode(); 335 if (ret < 0 ) 336 { 337 TEST(ret == 0); 338 printf ("Decode error in frame # %d",_frameCnt); 339 } 340 341 float encBytes = encodeCompleteCallback->EncodedBytes(); 342 if (encBytes == 0) 343 { 344 _numFramesDropped += 1; 345 //printf("frame #%d dropped \n", _frameCnt ); 346 } 347 else 348 { 349 // write frame to file 350 if (PrintI420VideoFrame(sourceFrame, _actualSourceFile) < 0) { 351 return -1; 352 } 353 } 354 355 _sumEncBytes += encBytes; 356 } 357 358 //END TEST 359 delete encodeCompleteCallback; 360 delete tmpBuffer; 361 362 return 0; 363 364 } 365 366 void 367 MediaOptTest::RTTest() 368 { 369 // will only calculate PSNR - not create output files for all 370 // SET UP 371 // Set bit rates 372 const float bitRateVec[] = {500, 1000, 2000,3000, 4000}; 373 //const float bitRateVec[] = {1000}; 374 // Set Packet loss values ([0,255]) 375 const double lossPctVec[] = {0.0*255, 0.0*255, 0.01*255, 0.01*255, 0.03*255, 0.03*255, 0.05*255, 0.05*255, 0.1*255, 0.1*255}; 376 const bool nackEnabledVec[] = {false , false, false, false, false, false, false, false , false, false}; 377 const bool fecEnabledVec[] = {false , true, false, true , false, true , false, true , false, true}; 378 // fec and nack are set according to the packet loss values 379 380 const float nBitrates = sizeof(bitRateVec)/sizeof(*bitRateVec); 381 const float nlossPct = sizeof(lossPctVec)/sizeof(*lossPctVec); 382 383 std::vector<const VideoSource*> sources; 384 std::vector<const VideoSource*>::iterator it; 385 386 sources.push_back(new const VideoSource(_inname, _width, _height)); 387 int numOfSrc = 1; 388 389 // constant settings (valid for entire run time) 390 _rttMS = 20; 391 _renderDelayMs = 0; 392 393 // same out name for all 394 _outname = test::OutputPath() + "RTMOTest_out.yuv"; 395 // actual source after frame dropping 396 _actualSourcename = test::OutputPath() + "RTMOTestSource.yuv"; 397 398 _codecName = "VP8"; // for now just this one - later iterate over all codec types 399 _log.open((test::OutputPath() + "/VCM_RTMediaOptLog.txt").c_str(), 400 std::fstream::out | std::fstream::app); 401 _outputRes=fopen((test::OutputPath() + "VCM_MediaOptResults.txt").c_str(), 402 "ab"); 403 404 //char filename[128]; 405 /* test settings end*/ 406 407 // START TEST 408 // iterate over test sequences 409 printf("\n****START TEST OVER ALL RUNS ****\n"); 410 int runCnt = 0; 411 for (it = sources.begin() ; it < sources.end(); it++) 412 { 413 414 // test set up 415 _inname = (*it)->GetFileName(); 416 _width = (*it)->GetWidth(); 417 _height = (*it)->GetHeight(); 418 _lengthSourceFrame = 3*_width*_height/2; 419 _frameRate = (*it)->GetFrameRate(); 420 //GeneralSetup(); 421 422 423 // iterate over all bit rates 424 for (int i = 0; i < nBitrates; i++) 425 { 426 _bitRate = static_cast<float>(bitRateVec[i]); 427 // iterate over all packet loss values 428 for (int j = 0; j < nlossPct; j++) 429 { 430 _lossRate = static_cast<float>(lossPctVec[j]); 431 _nackEnabled = static_cast<bool>(nackEnabledVec[j]); 432 _fecEnabled = static_cast<bool>(fecEnabledVec[j]); 433 434 runCnt++; 435 printf("run #%d out of %d \n", runCnt,(int)(nlossPct*nBitrates*numOfSrc)); 436 437 //printf("**FOR RUN: **%d %d %d %d \n",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate)); 438 439 /* 440 int ch = sprintf(filename,"../test_mediaOpt/RTMOTest_%d_%d_%d_%d.yuv",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate)); 441 _outname = filename; 442 443 printf("**FOR RUN: **%d %d %d %d \n",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate)); 444 */ 445 GeneralSetup(); 446 Perform(); 447 Print(1); 448 TearDown(); 449 450 printf("\n"); 451 //printf("**DONE WITH RUN: **%d %d %f %d \n",_nackEnabled,_fecEnabled,lossPctVec[j],int(_bitRate)); 452 // 453 454 }// end of packet loss loop 455 }// end of bit rate loop 456 delete *it; 457 }// end of video sequence loop 458 // at end of sequence 459 fclose(_outputRes); 460 printf("\nVCM Media Optimization Test: \n\n%i tests completed\n", vcmMacrosTests); 461 if (vcmMacrosErrors > 0) 462 { 463 printf("%i FAILED\n\n", vcmMacrosErrors); 464 } 465 else 466 { 467 printf("ALL PASSED\n\n"); 468 } 469 } 470 471 472 void 473 MediaOptTest::Print(int mode) 474 { 475 double ActualBitRate = 8.0 *( _sumEncBytes / (_frameCnt / _frameRate)); 476 double actualBitRate = ActualBitRate / 1000.0; 477 webrtc::test::QualityMetricsResult psnr; 478 I420PSNRFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width, 479 _height, &psnr); 480 481 (_log) << "VCM: Media Optimization Test Cycle Completed!" << std::endl; 482 (_log) << "Input file: " << _inname << std::endl; 483 (_log) << "Output file:" << _outname << std::endl; 484 ( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " << _bitRate << " kbps" << std::endl; 485 (_log) << "Error Reslience: NACK:" << _nackEnabled << "; FEC: " << _fecEnabled << std::endl; 486 (_log) << "Packet Loss applied= %f " << _lossRate << std::endl; 487 (_log) << _numFramesDropped << " FRames were dropped" << std::endl; 488 ( _log) << "PSNR: " << psnr.average << std::endl; 489 (_log) << std::endl; 490 491 if (_testType == 2) 492 { 493 fprintf(_outputRes,"************\n"); 494 fprintf(_outputRes,"\n\n\n"); 495 fprintf(_outputRes,"Actual bitrate: %f kbps\n", actualBitRate); 496 fprintf(_outputRes,"Target bitrate: %f kbps\n", _bitRate); 497 fprintf(_outputRes,"NACK: %s ",(_nackEnabled)?"true":"false"); 498 fprintf(_outputRes,"FEC: %s \n ",(_fecEnabled)?"true":"false"); 499 fprintf(_outputRes,"Packet loss applied = %f\n", _lossRate); 500 fprintf(_outputRes,"%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt); 501 fprintf(_outputRes,"PSNR: %f \n", psnr.average); 502 fprintf(_outputRes,"************\n"); 503 } 504 505 506 // 507 if (_testType == 1) 508 { 509 fprintf(_fpout,"************\n"); 510 fprintf(_fpout,"\n\n\n"); 511 fprintf(_fpout,"Actual bitrate: %f kbps\n", actualBitRate); 512 fprintf(_fpout,"Target bitrate: %f kbps\n", _bitRate); 513 fprintf(_fpout,"NACK: %s ",(_nackEnabled)?"true":"false"); 514 fprintf(_fpout,"FEC: %s \n ",(_fecEnabled)?"true":"false"); 515 fprintf(_fpout,"Packet loss applied = %f\n", _lossRate); 516 fprintf(_fpout,"%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt); 517 fprintf(_fpout,"PSNR: %f \n", psnr.average); 518 fprintf(_fpout,"************\n"); 519 520 int testNum1 = _testNum/(_numParRuns +1) + 1; 521 int testNum2 = _testNum%_numParRuns; 522 if (testNum2 == 0) testNum2 = _numParRuns; 523 fprintf(_fpout2,"%d %d %f %f %f %f \n",testNum1,testNum2,_bitRate,actualBitRate,_lossRate,psnr.average); 524 fclose(_fpinp); 525 _fpinp = fopen("dat_inp","wb"); 526 fprintf(_fpinp,"%f %f %d \n",_bitRate,_lossRate,_testNum); 527 } 528 // 529 530 531 if (mode == 1) 532 { 533 // print to screen 534 printf("\n\n\n"); 535 printf("Actual bitrate: %f kbps\n", actualBitRate); 536 printf("Target bitrate: %f kbps\n", _bitRate); 537 printf("NACK: %s ",(_nackEnabled)?"true":"false"); 538 printf("FEC: %s \n",(_fecEnabled)?"true":"false"); 539 printf("Packet loss applied = %f\n", _lossRate); 540 printf("%d frames were dropped, and total number of frames processed %d \n",_numFramesDropped,_frameCnt); 541 printf("PSNR: %f \n", psnr.average); 542 } 543 TEST(psnr.average > 10); // low becuase of possible frame dropping (need to verify that OK for all packet loss values/ rates) 544 } 545 546 void MediaOptTest::TearDown() { 547 delete _rtp; 548 _rtp = NULL; 549 delete _outgoingTransport; 550 _outgoingTransport = NULL; 551 delete _dataCallback; 552 _dataCallback = NULL; 553 _log.close(); 554 fclose(_sourceFile); 555 fclose(_decodedFile); 556 fclose(_actualSourceFile); 557 } 558