Home | History | Annotate | Download | only in test 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 // Implementation of Media Optimization Test
     12 // testing is done via the VCM module, no specific Media opt functionality.
     13 
     14 #include "webrtc/modules/video_coding/main/test/media_opt_test.h"
     15 
     16 #include <stdio.h>
     17 #include <string.h>
     18 #include <time.h>
     19 #include <vector>
     20 
     21 #include "webrtc/modules/rtp_rtcp/interface/rtp_receiver.h"
     22 #include "webrtc/modules/video_coding/main/interface/video_coding.h"
     23 #include "webrtc/modules/video_coding/main/test/test_macros.h"
     24 #include "webrtc/modules/video_coding/main/test/test_util.h"
     25 #include "webrtc/test/testsupport/fileutils.h"
     26 #include "webrtc/test/testsupport/metrics/video_metrics.h"
     27 
     28 using namespace webrtc;
     29 
     30 int MediaOptTest::RunTest(int testNum, CmdArgs& args)
     31 {
     32     Trace::CreateTrace();
     33     Trace::SetTraceFile((test::OutputPath() + "mediaOptTestTrace.txt").c_str());
     34     Trace::set_level_filter(webrtc::kTraceAll);
     35     VideoCodingModule* vcm = VideoCodingModule::Create();
     36     Clock* clock = Clock::GetRealTimeClock();
     37     MediaOptTest* mot = new MediaOptTest(vcm, clock);
     38     if (testNum == 0)
     39     { // regular
     40          mot->Setup(0, args);
     41          mot->GeneralSetup();
     42          mot->Perform();
     43          mot->Print(1);// print to screen
     44          mot->TearDown();
     45     }
     46     if (testNum == 1)
     47     {   // release test
     48         mot->Setup(0, args);
     49         mot->RTTest();
     50     }
     51     if (testNum == 2)
     52     { // release test, running from script
     53          mot->Setup(1, args);
     54          mot->GeneralSetup();
     55          mot->Perform();
     56          mot->Print(1);// print to screen
     57          mot->TearDown();
     58     }
     59 
     60     VideoCodingModule::Destroy(vcm);
     61     delete mot;
     62     Trace::ReturnTrace();
     63     return 0;
     64 
     65 }
     66 
     67 
     68 MediaOptTest::MediaOptTest(VideoCodingModule* vcm, Clock* clock)
     69     : _vcm(vcm),
     70       _rtp(NULL),
     71       _outgoingTransport(NULL),
     72       _dataCallback(NULL),
     73       _clock(clock),
     74       _width(0),
     75       _height(0),
     76       _lengthSourceFrame(0),
     77       _timeStamp(0),
     78       _frameRate(30.0f),
     79       _nackEnabled(false),
     80       _fecEnabled(false),
     81       _rttMS(0),
     82       _bitRate(300.0f),
     83       _lossRate(0.0f),
     84       _renderDelayMs(0),
     85       _frameCnt(0),
     86       _sumEncBytes(0),
     87       _numFramesDropped(0),
     88       _numberOfCores(4) {
     89 }
     90 
     91 MediaOptTest::~MediaOptTest() {
     92   delete _rtp;
     93 }
     94 
     95 void MediaOptTest::Setup(int testType, CmdArgs& args) {
     96     /*TEST USER SETTINGS*/
     97     // test parameters
     98     _inname = args.inputFile;
     99     if (args.outputFile == "")
    100         _outname = test::OutputPath() + "MOTest_out.vp8";
    101     else
    102         _outname = args.outputFile;
    103     // actual source after frame dropping
    104     _actualSourcename = test::OutputPath() + "MOTestSource.yuv";
    105     _codecName = args.codecName;
    106     _sendCodecType = args.codecType;
    107     _width = args.width;
    108     _height = args.height;
    109     _frameRate = args.frameRate;
    110     _bitRate = args.bitRate;
    111     _numberOfCores = 4;
    112 
    113     // error resilience
    114     _nackEnabled = false;
    115     _fecEnabled = true;
    116     _nackFecEnabled = false;
    117 
    118     _rttMS = 100;
    119     _lossRate = 0.00*255; // no packet loss
    120 
    121     _testType = testType;
    122 
    123     //For multiple runs with script
    124     if (_testType == 1)
    125     {
    126         float rateTest,lossTest;
    127         int numRuns;
    128         _fpinp = fopen("dat_inp","rb");
    129         _fpout = fopen("test_runs/dat_out","ab");
    130         _fpout2 = fopen("test_runs/dat_out2","ab");
    131         TEST(fscanf(_fpinp,"%f %f %d \n",&rateTest,&lossTest,&numRuns) > 0);
    132         _bitRate = rateTest;
    133         _lossRate = lossTest;
    134         _testNum = 0;
    135 
    136         // for bit rates: 500, 1000, 2000, 3000,4000
    137         // for loss rates: 0, 1, 3, 5, 10%
    138         _numParRuns = 25;
    139 
    140         _testNum = numRuns + 1;
    141         if (rateTest == 0.0) _lossRate = 0.0;
    142         else
    143         {
    144             if (rateTest == 4000)  //final bit rate
    145             {
    146                 if (lossTest == 0.1*255) _lossRate = 0.0;  //start at 1%
    147                 else
    148                     if (lossTest == 0.05*255) _lossRate = 0.1*255;  //final loss rate
    149                     else
    150                         if (lossTest == 0.0) _lossRate = 0.01*255;
    151                         else _lossRate = lossTest + 0.02*255;
    152             }
    153         }
    154 
    155         if (rateTest == 0.0 || rateTest == 4000) _bitRate = 500; //starting bit rate
    156         else
    157             if (rateTest == 500) _bitRate = 1000;
    158                 else _bitRate = rateTest +  1000;
    159     }
    160    //
    161 
    162     _renderDelayMs = 0;
    163     /* test settings end*/
    164 
    165    _lengthSourceFrame  = 3*_width*_height/2;
    166     _log.open((test::OutputPath() + "VCM_MediaOptLog.txt").c_str(),
    167               std::fstream::out | std::fstream::app);
    168 }
    169 
    170 void
    171 MediaOptTest::GeneralSetup()
    172 {
    173     uint32_t minPlayoutDelayMs = 0;
    174 
    175     if ((_sourceFile = fopen(_inname.c_str(), "rb")) == NULL)
    176     {
    177         printf("Cannot read file %s.\n", _inname.c_str());
    178         exit(1);
    179     }
    180 
    181     if ((_decodedFile = fopen(_outname.c_str(), "wb")) == NULL)
    182     {
    183         printf("Cannot read file %s.\n", _outname.c_str());
    184         exit(1);
    185     }
    186 
    187     if ((_actualSourceFile = fopen(_actualSourcename.c_str(), "wb")) == NULL)
    188     {
    189         printf("Cannot read file %s.\n", _actualSourcename.c_str());
    190         exit(1);
    191     }
    192     if (_vcm->InitializeReceiver() < 0)
    193     {
    194         exit(1);
    195     }
    196     if (_vcm->InitializeSender())
    197     {
    198         exit(1);
    199     }
    200     _outgoingTransport = new RTPSendCompleteCallback(_clock);
    201     _dataCallback = new RtpDataCallback(_vcm);
    202 
    203     RtpRtcp::Configuration configuration;
    204     configuration.id = 1;
    205     configuration.audio = false;
    206     configuration.outgoing_transport = _outgoingTransport;
    207     _rtp = RtpRtcp::CreateRtpRtcp(configuration);
    208 
    209     _outgoingTransport->SetRtpModule(_rtp);
    210 
    211     // Registering codecs for the RTP module
    212 
    213     // Register receive and send payload
    214     VideoCodec video_codec;
    215     strncpy(video_codec.plName, "VP8", 32);
    216     video_codec.plType = VCM_VP8_PAYLOAD_TYPE;
    217     rtp_receiver_->RegisterReceivePayload(video_codec.plName,
    218                                           video_codec.plType,
    219                                           90000,
    220                                           0,
    221                                           video_codec.maxBitrate);
    222     _rtp->RegisterSendPayload(video_codec);
    223 
    224     strncpy(video_codec.plName, "ULPFEC", 32);
    225     video_codec.plType = VCM_ULPFEC_PAYLOAD_TYPE;
    226     rtp_receiver_->RegisterReceivePayload(video_codec.plName,
    227                                           video_codec.plType,
    228                                           90000,
    229                                           0,
    230                                           video_codec.maxBitrate);
    231     _rtp->RegisterSendPayload(video_codec);
    232 
    233     strncpy(video_codec.plName, "RED", 32);
    234     video_codec.plType = VCM_RED_PAYLOAD_TYPE;
    235     rtp_receiver_->RegisterReceivePayload(video_codec.plName,
    236                                           video_codec.plType,
    237                                           90000,
    238                                           0,
    239                                           video_codec.maxBitrate);
    240     _rtp->RegisterSendPayload(video_codec);
    241 
    242     if (_nackFecEnabled == 1)
    243         _rtp->SetGenericFECStatus(_nackFecEnabled, VCM_RED_PAYLOAD_TYPE,
    244                 VCM_ULPFEC_PAYLOAD_TYPE);
    245     else
    246         _rtp->SetGenericFECStatus(_fecEnabled, VCM_RED_PAYLOAD_TYPE,
    247                 VCM_ULPFEC_PAYLOAD_TYPE);
    248 
    249     // VCM: Registering codecs
    250     VideoCodec sendCodec;
    251     _vcm->InitializeSender();
    252     _vcm->InitializeReceiver();
    253     int32_t numberOfCodecs = _vcm->NumberOfCodecs();
    254     if (numberOfCodecs < 1)
    255     {
    256         exit(1);
    257     }
    258 
    259     if (_vcm->Codec(_sendCodecType, &sendCodec) != 0)
    260     {
    261         printf("Unknown codec\n");
    262         exit(1);
    263     }
    264     // register codec
    265     sendCodec.startBitrate = (int) _bitRate;
    266     sendCodec.height = _height;
    267     sendCodec.width = _width;
    268     sendCodec.maxFramerate = (uint8_t)_frameRate;
    269     _vcm->RegisterSendCodec(&sendCodec, _numberOfCores, 1440);
    270     _vcm->RegisterReceiveCodec(&sendCodec, _numberOfCores); // same settings for encode and decode
    271 
    272     _vcm->SetRenderDelay(_renderDelayMs);
    273     _vcm->SetMinimumPlayoutDelay(minPlayoutDelayMs);
    274 }
    275 // The following test shall be conducted under release tests
    276 
    277 
    278 
    279 int32_t
    280 MediaOptTest::Perform()
    281 {
    282     VCMDecodeCompleteCallback receiveCallback(_decodedFile);
    283 
    284     VCMRTPEncodeCompleteCallback* encodeCompleteCallback = new VCMRTPEncodeCompleteCallback(_rtp);
    285     _vcm->RegisterTransportCallback(encodeCompleteCallback);
    286     encodeCompleteCallback->SetCodecType(ConvertCodecType(_codecName.c_str()));
    287     encodeCompleteCallback->SetFrameDimensions(_width, _height);
    288 
    289     // callback settings
    290     VideoProtectionCallback  protectionCallback;
    291     protectionCallback.RegisterRtpModule(_rtp);
    292     _vcm->RegisterProtectionCallback(&protectionCallback);
    293 
    294     // set error resilience / test parameters:
    295     _outgoingTransport->SetLossPct(_lossRate);
    296     if (_nackFecEnabled == 1) {
    297         _vcm->SetVideoProtection(kProtectionNackFEC, _nackFecEnabled);
    298     } else {
    299         _vcm->SetVideoProtection(kProtectionNack, _nackEnabled);
    300         _vcm->SetVideoProtection(kProtectionFEC, _fecEnabled);
    301     }
    302 
    303     // START TEST
    304     I420VideoFrame sourceFrame;
    305     uint8_t* tmpBuffer = new uint8_t[_lengthSourceFrame];
    306     _vcm->SetChannelParameters(static_cast<uint32_t>(1000 * _bitRate),
    307                                (uint8_t)_lossRate, _rttMS);
    308     _vcm->RegisterReceiveCallback(&receiveCallback);
    309 
    310     _frameCnt  = 0;
    311     _sumEncBytes = 0.0;
    312     _numFramesDropped = 0;
    313     int half_width = (_width + 1) / 2;
    314     int half_height = (_height + 1) / 2;
    315     int size_y = _width * _height;
    316     int size_uv = half_width * half_height;
    317 
    318     while (feof(_sourceFile)== 0)
    319     {
    320         TEST(fread(tmpBuffer, 1, _lengthSourceFrame, _sourceFile) > 0);
    321         _frameCnt++;
    322         sourceFrame.CreateFrame(size_y, tmpBuffer,
    323                                 size_uv, tmpBuffer + size_y,
    324                                 size_uv, tmpBuffer + size_y + size_uv,
    325                                 _width, _height,
    326                                 _width, half_width, half_width);
    327         _timeStamp += (uint32_t)(9e4 / static_cast<float>(_frameRate));
    328         sourceFrame.set_timestamp(_timeStamp);
    329         TEST(_vcm->AddVideoFrame(sourceFrame) == VCM_OK);
    330         // inform RTP Module of error resilience features
    331         //_rtp->SetFECCodeRate(protectionCallback.FECKeyRate(),protectionCallback.FECDeltaRate());
    332         //_rtp->SetNACKStatus(protectionCallback.NACKMethod());
    333 
    334         int32_t ret = _vcm->Decode();
    335         if (ret < 0 )
    336         {
    337             TEST(ret == 0);
    338             printf ("Decode error in frame # %d",_frameCnt);
    339         }
    340 
    341         float encBytes = encodeCompleteCallback->EncodedBytes();
    342         if (encBytes == 0)
    343         {
    344             _numFramesDropped += 1;
    345             //printf("frame #%d dropped \n", _frameCnt );
    346         }
    347         else
    348         {
    349           // write frame to file
    350           if (PrintI420VideoFrame(sourceFrame, _actualSourceFile) < 0) {
    351             return -1;
    352           }
    353         }
    354 
    355         _sumEncBytes += encBytes;
    356     }
    357 
    358     //END TEST
    359     delete encodeCompleteCallback;
    360     delete tmpBuffer;
    361 
    362 return 0;
    363 
    364 }
    365 
    366 void
    367 MediaOptTest::RTTest()
    368 {
    369     // will only calculate PSNR - not create output files for all
    370     // SET UP
    371     // Set bit rates
    372     const float bitRateVec[] = {500, 1000, 2000,3000, 4000};
    373     //const float bitRateVec[] = {1000};
    374     // Set Packet loss values ([0,255])
    375     const double lossPctVec[]     = {0.0*255, 0.0*255, 0.01*255, 0.01*255, 0.03*255, 0.03*255, 0.05*255, 0.05*255, 0.1*255, 0.1*255};
    376     const bool  nackEnabledVec[] = {false  , false, false, false, false, false, false, false , false, false};
    377     const bool  fecEnabledVec[]  = {false  , true,  false, true , false, true , false, true , false, true};
    378     // fec and nack are set according to the packet loss values
    379 
    380     const float nBitrates = sizeof(bitRateVec)/sizeof(*bitRateVec);
    381     const float nlossPct = sizeof(lossPctVec)/sizeof(*lossPctVec);
    382 
    383     std::vector<const VideoSource*> sources;
    384     std::vector<const VideoSource*>::iterator it;
    385 
    386     sources.push_back(new const VideoSource(_inname, _width, _height));
    387     int numOfSrc = 1;
    388 
    389     // constant settings (valid for entire run time)
    390     _rttMS = 20;
    391     _renderDelayMs = 0;
    392 
    393     // same out name for all
    394     _outname = test::OutputPath() + "RTMOTest_out.yuv";
    395     // actual source after frame dropping
    396     _actualSourcename = test::OutputPath() + "RTMOTestSource.yuv";
    397 
    398     _codecName = "VP8";  // for now just this one - later iterate over all codec types
    399     _log.open((test::OutputPath() + "/VCM_RTMediaOptLog.txt").c_str(),
    400               std::fstream::out | std::fstream::app);
    401     _outputRes=fopen((test::OutputPath() + "VCM_MediaOptResults.txt").c_str(),
    402                      "ab");
    403 
    404     //char filename[128];
    405     /* test settings end*/
    406 
    407     // START TEST
    408     // iterate over test sequences
    409     printf("\n****START TEST OVER ALL RUNS ****\n");
    410     int runCnt = 0;
    411     for (it = sources.begin() ; it < sources.end(); it++)
    412     {
    413 
    414         // test set up
    415         _inname = (*it)->GetFileName();
    416         _width  = (*it)->GetWidth();
    417         _height = (*it)->GetHeight();
    418         _lengthSourceFrame  = 3*_width*_height/2;
    419         _frameRate = (*it)->GetFrameRate();
    420          //GeneralSetup();
    421 
    422 
    423         // iterate over all bit rates
    424         for (int i = 0; i < nBitrates; i++)
    425         {
    426            _bitRate = static_cast<float>(bitRateVec[i]);
    427             // iterate over all packet loss values
    428             for (int j = 0; j < nlossPct; j++)
    429             {
    430                  _lossRate = static_cast<float>(lossPctVec[j]);
    431                  _nackEnabled = static_cast<bool>(nackEnabledVec[j]);
    432                  _fecEnabled = static_cast<bool>(fecEnabledVec[j]);
    433 
    434                  runCnt++;
    435                  printf("run #%d out of %d \n", runCnt,(int)(nlossPct*nBitrates*numOfSrc));
    436 
    437                 //printf("**FOR RUN: **%d %d %d %d \n",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate));
    438 
    439                  /*
    440                  int ch = sprintf(filename,"../test_mediaOpt/RTMOTest_%d_%d_%d_%d.yuv",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate));
    441                 _outname = filename;
    442 
    443                 printf("**FOR RUN: **%d %d %d %d \n",_nackEnabled,_fecEnabled,int(lossPctVec[j]),int(_bitRate));
    444                */
    445                  GeneralSetup();
    446                  Perform();
    447                  Print(1);
    448                  TearDown();
    449 
    450                  printf("\n");
    451                   //printf("**DONE WITH RUN: **%d %d %f %d \n",_nackEnabled,_fecEnabled,lossPctVec[j],int(_bitRate));
    452                  //
    453 
    454             }// end of packet loss loop
    455         }// end of bit rate loop
    456         delete *it;
    457     }// end of video sequence loop
    458     // at end of sequence
    459     fclose(_outputRes);
    460     printf("\nVCM Media Optimization Test: \n\n%i tests completed\n", vcmMacrosTests);
    461     if (vcmMacrosErrors > 0)
    462     {
    463         printf("%i FAILED\n\n", vcmMacrosErrors);
    464     }
    465     else
    466     {
    467         printf("ALL PASSED\n\n");
    468     }
    469 }
    470 
    471 
    472 void
    473 MediaOptTest::Print(int mode)
    474 {
    475     double ActualBitRate =  8.0 *( _sumEncBytes / (_frameCnt / _frameRate));
    476     double actualBitRate = ActualBitRate / 1000.0;
    477     webrtc::test::QualityMetricsResult psnr;
    478     I420PSNRFromFiles(_actualSourcename.c_str(), _outname.c_str(), _width,
    479                       _height, &psnr);
    480 
    481     (_log) << "VCM: Media Optimization Test Cycle Completed!" << std::endl;
    482     (_log) << "Input file: " << _inname << std::endl;
    483     (_log) << "Output file:" << _outname << std::endl;
    484     ( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " << _bitRate << " kbps" << std::endl;
    485     (_log) << "Error Reslience: NACK:" << _nackEnabled << "; FEC: " << _fecEnabled << std::endl;
    486     (_log) << "Packet Loss applied= %f " << _lossRate << std::endl;
    487     (_log) << _numFramesDropped << " FRames were dropped" << std::endl;
    488      ( _log) << "PSNR: " << psnr.average << std::endl;
    489     (_log) << std::endl;
    490 
    491     if (_testType == 2)
    492     {
    493         fprintf(_outputRes,"************\n");
    494         fprintf(_outputRes,"\n\n\n");
    495         fprintf(_outputRes,"Actual bitrate: %f kbps\n", actualBitRate);
    496         fprintf(_outputRes,"Target bitrate: %f kbps\n", _bitRate);
    497         fprintf(_outputRes,"NACK: %s  ",(_nackEnabled)?"true":"false");
    498         fprintf(_outputRes,"FEC: %s \n ",(_fecEnabled)?"true":"false");
    499         fprintf(_outputRes,"Packet loss applied = %f\n", _lossRate);
    500         fprintf(_outputRes,"%d frames were dropped, and total number of frames processed %d  \n",_numFramesDropped,_frameCnt);
    501         fprintf(_outputRes,"PSNR: %f \n", psnr.average);
    502         fprintf(_outputRes,"************\n");
    503     }
    504 
    505 
    506     //
    507     if (_testType == 1)
    508     {
    509         fprintf(_fpout,"************\n");
    510         fprintf(_fpout,"\n\n\n");
    511         fprintf(_fpout,"Actual bitrate: %f kbps\n", actualBitRate);
    512         fprintf(_fpout,"Target bitrate: %f kbps\n", _bitRate);
    513         fprintf(_fpout,"NACK: %s  ",(_nackEnabled)?"true":"false");
    514         fprintf(_fpout,"FEC: %s \n ",(_fecEnabled)?"true":"false");
    515         fprintf(_fpout,"Packet loss applied = %f\n", _lossRate);
    516         fprintf(_fpout,"%d frames were dropped, and total number of frames processed %d  \n",_numFramesDropped,_frameCnt);
    517         fprintf(_fpout,"PSNR: %f \n", psnr.average);
    518         fprintf(_fpout,"************\n");
    519 
    520         int testNum1 = _testNum/(_numParRuns +1) + 1;
    521         int testNum2 = _testNum%_numParRuns;
    522         if (testNum2 == 0) testNum2 = _numParRuns;
    523         fprintf(_fpout2,"%d %d %f %f %f %f \n",testNum1,testNum2,_bitRate,actualBitRate,_lossRate,psnr.average);
    524         fclose(_fpinp);
    525         _fpinp = fopen("dat_inp","wb");
    526         fprintf(_fpinp,"%f %f %d \n",_bitRate,_lossRate,_testNum);
    527     }
    528     //
    529 
    530 
    531     if (mode == 1)
    532     {
    533         // print to screen
    534         printf("\n\n\n");
    535         printf("Actual bitrate: %f kbps\n", actualBitRate);
    536         printf("Target bitrate: %f kbps\n", _bitRate);
    537         printf("NACK: %s  ",(_nackEnabled)?"true":"false");
    538         printf("FEC: %s \n",(_fecEnabled)?"true":"false");
    539         printf("Packet loss applied = %f\n", _lossRate);
    540         printf("%d frames were dropped, and total number of frames processed %d  \n",_numFramesDropped,_frameCnt);
    541         printf("PSNR: %f \n", psnr.average);
    542     }
    543     TEST(psnr.average > 10); // low becuase of possible frame dropping (need to verify that OK for all packet loss values/ rates)
    544 }
    545 
    546 void MediaOptTest::TearDown() {
    547   delete _rtp;
    548   _rtp = NULL;
    549   delete _outgoingTransport;
    550   _outgoingTransport = NULL;
    551   delete _dataCallback;
    552   _dataCallback = NULL;
    553   _log.close();
    554   fclose(_sourceFile);
    555   fclose(_decodedFile);
    556   fclose(_actualSourceFile);
    557 }
    558