Home | History | Annotate | Download | only in test 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/modules/video_coding/main/test/quality_modes_test.h"
     12 
     13 #include <iostream>
     14 #include <sstream>
     15 #include <string>
     16 #include <time.h>
     17 
     18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     19 #include "webrtc/modules/video_coding/main/interface/video_coding.h"
     20 #include "webrtc/modules/video_coding/main/test/test_callbacks.h"
     21 #include "webrtc/modules/video_coding/main/test/test_macros.h"
     22 #include "webrtc/modules/video_coding/main/test/test_util.h"
     23 #include "webrtc/system_wrappers/interface/clock.h"
     24 #include "webrtc/system_wrappers/interface/data_log.h"
     25 #include "webrtc/system_wrappers/interface/data_log.h"
     26 #include "webrtc/test/testsupport/fileutils.h"
     27 #include "webrtc/test/testsupport/metrics/video_metrics.h"
     28 
     29 using namespace webrtc;
     30 
     31 int qualityModeTest(const CmdArgs& args)
     32 {
     33   SimulatedClock clock(0);
     34   NullEventFactory event_factory;
     35   VideoCodingModule* vcm = VideoCodingModule::Create(&clock, &event_factory);
     36   QualityModesTest QMTest(vcm, &clock);
     37   QMTest.Perform(args);
     38   VideoCodingModule::Destroy(vcm);
     39   return 0;
     40 }
     41 
     42 QualityModesTest::QualityModesTest(VideoCodingModule* vcm,
     43                                    Clock* clock):
     44 NormalTest(vcm, clock),
     45 _vpm()
     46 {
     47     //
     48 }
     49 
     50 QualityModesTest::~QualityModesTest()
     51 {
     52     //
     53 }
     54 
     55 void
     56 QualityModesTest::Setup(const CmdArgs& args)
     57 {
     58   NormalTest::Setup(args);
     59   _inname = args.inputFile;
     60   _outname = args.outputFile;
     61   fv_outfilename_ = args.fv_outputfile;
     62 
     63   filename_testvideo_ =
     64       _inname.substr(_inname.find_last_of("\\/") + 1,_inname.length());
     65 
     66   _encodedName = test::OutputPath() + "encoded_qmtest.yuv";
     67 
     68   //NATIVE/SOURCE VALUES
     69   _nativeWidth = args.width;
     70   _nativeHeight = args.height;
     71   _nativeFrameRate =args.frameRate;
     72 
     73   //TARGET/ENCODER VALUES
     74   _width = args.width;
     75   _height = args.height;
     76   _frameRate = args.frameRate;
     77 
     78   _bitRate = args.bitRate;
     79 
     80   _flagSSIM = true;
     81 
     82   _lengthSourceFrame  = 3*_nativeWidth*_nativeHeight/2;
     83 
     84   if ((_sourceFile = fopen(_inname.c_str(), "rb")) == NULL)
     85   {
     86     printf("Cannot read file %s.\n", _inname.c_str());
     87     exit(1);
     88   }
     89   if ((_encodedFile = fopen(_encodedName.c_str(), "wb")) == NULL)
     90   {
     91     printf("Cannot write encoded file.\n");
     92     exit(1);
     93   }
     94   if ((_decodedFile = fopen(_outname.c_str(),  "wb")) == NULL)
     95   {
     96     printf("Cannot write file %s.\n", _outname.c_str());
     97     exit(1);
     98   }
     99 
    100   DataLog::CreateLog();
    101 
    102   feature_table_name_ = fv_outfilename_;
    103 
    104   DataLog::AddTable(feature_table_name_);
    105 
    106   DataLog::AddColumn(feature_table_name_, "motion magnitude", 1);
    107   DataLog::AddColumn(feature_table_name_, "spatial prediction error", 1);
    108   DataLog::AddColumn(feature_table_name_, "spatial pred err horizontal", 1);
    109   DataLog::AddColumn(feature_table_name_, "spatial pred err vertical", 1);
    110   DataLog::AddColumn(feature_table_name_, "width", 1);
    111   DataLog::AddColumn(feature_table_name_, "height", 1);
    112   DataLog::AddColumn(feature_table_name_, "num pixels", 1);
    113   DataLog::AddColumn(feature_table_name_, "frame rate", 1);
    114   DataLog::AddColumn(feature_table_name_, "num frames since drop", 1);
    115 
    116   _log.open((test::OutputPath() + "TestLog.txt").c_str(),
    117             std::fstream::out | std::fstream::app);
    118 }
    119 
    120 void
    121 QualityModesTest::Print()
    122 {
    123   std::cout << "Quality Modes Test Completed!" << std::endl;
    124   (_log) << "Quality Modes Test Completed!" << std::endl;
    125   (_log) << "Input file: " << _inname << std::endl;
    126   (_log) << "Output file: " << _outname << std::endl;
    127   (_log) << "Total run time: " << _testTotalTime << std::endl;
    128   printf("Total run time: %f s \n", _testTotalTime);
    129   double ActualBitRate = 8.0*( _sumEncBytes / (_frameCnt / _nativeFrameRate));
    130   double actualBitRate = ActualBitRate / 1000.0;
    131   double avgEncTime = _totalEncodeTime / _frameCnt;
    132   double avgDecTime = _totalDecodeTime / _frameCnt;
    133   webrtc::test::QualityMetricsResult psnr,ssim;
    134   I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth,
    135                     _nativeHeight, &psnr);
    136   printf("Actual bitrate: %f kbps\n", actualBitRate);
    137   printf("Target bitrate: %f kbps\n", _bitRate);
    138   ( _log) << "Actual bitrate: " << actualBitRate<< " kbps\tTarget: " <<
    139       _bitRate << " kbps" << std::endl;
    140   printf("Average encode time: %f s\n", avgEncTime);
    141   ( _log) << "Average encode time: " << avgEncTime << " s" << std::endl;
    142   printf("Average decode time: %f s\n", avgDecTime);
    143   ( _log) << "Average decode time: " << avgDecTime << " s" << std::endl;
    144   printf("PSNR: %f \n", psnr.average);
    145   printf("**Number of frames dropped in VPM***%d \n",_numFramesDroppedVPM);
    146   ( _log) << "PSNR: " << psnr.average << std::endl;
    147   if (_flagSSIM == 1)
    148   {
    149     printf("***computing SSIM***\n");
    150     I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), _nativeWidth,
    151                       _nativeHeight, &ssim);
    152     printf("SSIM: %f \n", ssim.average);
    153   }
    154   (_log) << std::endl;
    155 
    156   printf("\nVCM Quality Modes Test: \n\n%i tests completed\n", vcmMacrosTests);
    157   if (vcmMacrosErrors > 0)
    158   {
    159     printf("%i FAILED\n\n", vcmMacrosErrors);
    160   }
    161   else
    162   {
    163     printf("ALL PASSED\n\n");
    164   }
    165 }
    166 void
    167 QualityModesTest::Teardown()
    168 {
    169   _log.close();
    170   fclose(_sourceFile);
    171   fclose(_decodedFile);
    172   fclose(_encodedFile);
    173   return;
    174 }
    175 
    176 int32_t
    177 QualityModesTest::Perform(const CmdArgs& args)
    178 {
    179   Setup(args);
    180   // changing bit/frame rate during the test
    181   const float bitRateUpdate[] = {1000};
    182   const float frameRateUpdate[] = {30};
    183   // frame num at which an update will occur
    184   const int updateFrameNum[] = {10000};
    185 
    186   uint32_t numChanges = sizeof(updateFrameNum)/sizeof(*updateFrameNum);
    187   uint8_t change = 0;// change counter
    188 
    189   _vpm = VideoProcessingModule::Create(1);
    190   EventWrapper* waitEvent = EventWrapper::Create();
    191   VideoCodec codec;//both send and receive
    192   _vcm->InitializeReceiver();
    193   _vcm->InitializeSender();
    194   int32_t NumberOfCodecs = _vcm->NumberOfCodecs();
    195   for (int i = 0; i < NumberOfCodecs; i++)
    196   {
    197     _vcm->Codec(i, &codec);
    198     if(strncmp(codec.plName,"VP8" , 5) == 0)
    199     {
    200       codec.startBitrate = (int)_bitRate;
    201       codec.maxFramerate = (uint8_t) _frameRate;
    202       codec.width = (uint16_t)_width;
    203       codec.height = (uint16_t)_height;
    204       codec.codecSpecific.VP8.frameDroppingOn = false;
    205 
    206       // Will also set and init the desired codec
    207       TEST(_vcm->RegisterSendCodec(&codec, 2, 1440) == VCM_OK);
    208       i = NumberOfCodecs;
    209     }
    210   }
    211 
    212   // register a decoder (same codec for decoder and encoder )
    213   TEST(_vcm->RegisterReceiveCodec(&codec, 2) == VCM_OK);
    214   /* Callback Settings */
    215   VCMQMDecodeCompleCallback  _decodeCallback(
    216       _decodedFile, _nativeFrameRate, feature_table_name_);
    217   _vcm->RegisterReceiveCallback(&_decodeCallback);
    218   VCMNTEncodeCompleteCallback   _encodeCompleteCallback(_encodedFile, *this);
    219   _vcm->RegisterTransportCallback(&_encodeCompleteCallback);
    220   // encode and decode with the same vcm
    221   _encodeCompleteCallback.RegisterReceiverVCM(_vcm);
    222 
    223   //quality modes callback
    224   QMTestVideoSettingsCallback QMCallback;
    225   QMCallback.RegisterVCM(_vcm);
    226   QMCallback.RegisterVPM(_vpm);
    227   //_vcm->RegisterVideoQMCallback(&QMCallback);
    228 
    229   ///////////////////////
    230   /// Start Test
    231   ///////////////////////
    232   _vpm->EnableTemporalDecimation(true);
    233   _vpm->EnableContentAnalysis(true);
    234   _vpm->SetInputFrameResampleMode(kFastRescaling);
    235 
    236   // disabling internal VCM frame dropper
    237   _vcm->EnableFrameDropper(false);
    238 
    239   I420VideoFrame sourceFrame;
    240   I420VideoFrame *decimatedFrame = NULL;
    241   uint8_t* tmpBuffer = new uint8_t[_lengthSourceFrame];
    242   double startTime = clock()/(double)CLOCKS_PER_SEC;
    243   _vcm->SetChannelParameters(static_cast<uint32_t>(1000 * _bitRate), 0, 0);
    244 
    245   SendStatsTest sendStats;
    246   sendStats.set_framerate(static_cast<uint32_t>(_frameRate));
    247   sendStats.set_bitrate(1000 * _bitRate);
    248   _vcm->RegisterSendStatisticsCallback(&sendStats);
    249 
    250   VideoContentMetrics* contentMetrics = NULL;
    251   // setting user frame rate
    252   // for starters: keeping native values:
    253   _vpm->SetTargetResolution(_width, _height,
    254                             (uint32_t)(_frameRate+ 0.5f));
    255   _decodeCallback.SetOriginalFrameDimensions(_nativeWidth, _nativeHeight);
    256 
    257   //tmp  - disabling VPM frame dropping
    258   _vpm->EnableTemporalDecimation(false);
    259 
    260   int32_t ret = 0;
    261   _numFramesDroppedVPM = 0;
    262 
    263   do {
    264     if (fread(tmpBuffer, 1, _lengthSourceFrame, _sourceFile) > 0) {
    265       _frameCnt++;
    266       int size_y = _nativeWidth * _nativeHeight;
    267       int size_uv = ((_nativeWidth + 1) / 2) * ((_nativeHeight  + 1) / 2);
    268       sourceFrame.CreateFrame(size_y, tmpBuffer,
    269                               size_uv, tmpBuffer + size_y,
    270                               size_uv, tmpBuffer + size_y + size_uv,
    271                               _nativeWidth, _nativeHeight,
    272                               _nativeWidth, (_nativeWidth + 1) / 2,
    273                               (_nativeWidth + 1) / 2);
    274 
    275       _timeStamp +=
    276           (uint32_t)(9e4 / static_cast<float>(codec.maxFramerate));
    277       sourceFrame.set_timestamp(_timeStamp);
    278 
    279       ret = _vpm->PreprocessFrame(sourceFrame, &decimatedFrame);
    280       if (ret  == 1)
    281       {
    282         printf("VD: frame drop %d \n",_frameCnt);
    283         _numFramesDroppedVPM += 1;
    284         continue; // frame drop
    285       }
    286       else if (ret < 0)
    287       {
    288         printf("Error in PreprocessFrame: %d\n", ret);
    289         //exit(1);
    290       }
    291       // Frame was not re-sampled => use original.
    292       if (decimatedFrame == NULL)
    293       {
    294         decimatedFrame = &sourceFrame;
    295       }
    296       contentMetrics = _vpm->ContentMetrics();
    297       if (contentMetrics == NULL)
    298       {
    299         printf("error: contentMetrics = NULL\n");
    300       }
    301 
    302       // counting only encoding time
    303       _encodeTimes[int(sourceFrame.timestamp())] =
    304           clock()/(double)CLOCKS_PER_SEC;
    305 
    306       int32_t ret = _vcm->AddVideoFrame(*decimatedFrame, contentMetrics);
    307 
    308       _totalEncodeTime += clock()/(double)CLOCKS_PER_SEC -
    309           _encodeTimes[int(sourceFrame.timestamp())];
    310 
    311       if (ret < 0)
    312       {
    313         printf("Error in AddFrame: %d\n", ret);
    314         //exit(1);
    315       }
    316 
    317       // Same timestamp value for encode and decode
    318       _decodeTimes[int(sourceFrame.timestamp())] =
    319           clock()/(double)CLOCKS_PER_SEC;
    320       ret = _vcm->Decode();
    321 
    322       _totalDecodeTime += clock()/(double)CLOCKS_PER_SEC -
    323           _decodeTimes[int(sourceFrame.timestamp())];
    324 
    325       if (ret < 0)
    326       {
    327         printf("Error in Decode: %d\n", ret);
    328         //exit(1);
    329       }
    330       if (_vcm->TimeUntilNextProcess() <= 0)
    331       {
    332         _vcm->Process();
    333       }
    334       // mimicking setTargetRates - update every 1 sec
    335       // this will trigger QMSelect
    336       if (_frameCnt%((int)_frameRate) == 0)
    337       {
    338         _vcm->SetChannelParameters(static_cast<uint32_t>(1000 * _bitRate), 0,
    339                                    1);
    340       }
    341 
    342       // check for bit rate update
    343       if (change < numChanges && _frameCnt == updateFrameNum[change])
    344       {
    345         _bitRate = bitRateUpdate[change];
    346         _frameRate = frameRateUpdate[change];
    347         codec.startBitrate = (int)_bitRate;
    348         codec.maxFramerate = (uint8_t) _frameRate;
    349         // Will also set and init the desired codec
    350         TEST(_vcm->RegisterSendCodec(&codec, 2, 1440) == VCM_OK);
    351         change++;
    352       }
    353 
    354       DataLog::InsertCell(feature_table_name_, "motion magnitude",
    355                           contentMetrics->motion_magnitude);
    356       DataLog::InsertCell(feature_table_name_, "spatial prediction error",
    357                           contentMetrics->spatial_pred_err);
    358       DataLog::InsertCell(feature_table_name_, "spatial pred err horizontal",
    359                           contentMetrics->spatial_pred_err_h);
    360       DataLog::InsertCell(feature_table_name_, "spatial pred err vertical",
    361                           contentMetrics->spatial_pred_err_v);
    362 
    363       DataLog::InsertCell(feature_table_name_, "width", _nativeHeight);
    364       DataLog::InsertCell(feature_table_name_, "height", _nativeWidth);
    365 
    366       DataLog::InsertCell(feature_table_name_, "num pixels",
    367                           _nativeHeight * _nativeWidth);
    368 
    369       DataLog::InsertCell(feature_table_name_, "frame rate", _nativeFrameRate);
    370       DataLog::NextRow(feature_table_name_);
    371 
    372       static_cast<SimulatedClock*>(_clock)->AdvanceTimeMilliseconds(
    373           1000 / _nativeFrameRate);
    374   }
    375 
    376   } while (feof(_sourceFile) == 0);
    377   _decodeCallback.WriteEnd(_frameCnt);
    378 
    379 
    380   double endTime = clock()/(double)CLOCKS_PER_SEC;
    381   _testTotalTime = endTime - startTime;
    382   _sumEncBytes = _encodeCompleteCallback.EncodedBytes();
    383 
    384   delete tmpBuffer;
    385   delete waitEvent;
    386   _vpm->Reset();
    387   Teardown();
    388   Print();
    389   VideoProcessingModule::Destroy(_vpm);
    390   return 0;
    391 }
    392 
    393 // implementing callback to be called from
    394 // VCM to update VPM of frame rate and size
    395 QMTestVideoSettingsCallback::QMTestVideoSettingsCallback():
    396 _vpm(NULL),
    397 _vcm(NULL)
    398 {
    399   //
    400 }
    401 
    402 void
    403 QMTestVideoSettingsCallback::RegisterVPM(VideoProcessingModule *vpm)
    404 {
    405   _vpm = vpm;
    406 }
    407 void
    408 QMTestVideoSettingsCallback::RegisterVCM(VideoCodingModule *vcm)
    409 {
    410   _vcm = vcm;
    411 }
    412 
    413 bool
    414 QMTestVideoSettingsCallback::Updated()
    415 {
    416   if (_updated)
    417   {
    418     _updated = false;
    419     return true;
    420   }
    421   return false;
    422 }
    423 
    424 int32_t
    425 QMTestVideoSettingsCallback::SetVideoQMSettings(const uint32_t frameRate,
    426                                                 const uint32_t width,
    427                                                 const uint32_t height)
    428 {
    429   int32_t retVal = 0;
    430   printf("QM updates: W = %d, H = %d, FR = %d, \n", width, height, frameRate);
    431   retVal = _vpm->SetTargetResolution(width, height, frameRate);
    432   //Initialize codec with new values - is this the best place to do it?
    433   if (!retVal)
    434   {
    435     // first get current settings
    436     VideoCodec currentCodec;
    437     _vcm->SendCodec(&currentCodec);
    438     // now set new values:
    439     currentCodec.height = (uint16_t)height;
    440     currentCodec.width = (uint16_t)width;
    441     currentCodec.maxFramerate = (uint8_t)frameRate;
    442 
    443     // re-register encoder
    444     retVal = _vcm->RegisterSendCodec(&currentCodec, 2, 1440);
    445     _updated = true;
    446   }
    447 
    448   return retVal;
    449 }
    450 
    451 // Decoded Frame Callback Implementation
    452 VCMQMDecodeCompleCallback::VCMQMDecodeCompleCallback(
    453     FILE* decodedFile, int frame_rate, std::string feature_table_name):
    454 _decodedFile(decodedFile),
    455 _decodedBytes(0),
    456 //_test(test),
    457 _origWidth(0),
    458 _origHeight(0),
    459 _decWidth(0),
    460 _decHeight(0),
    461 //_interpolator(NULL),
    462 _decBuffer(NULL),
    463 _frameCnt(0),
    464 frame_rate_(frame_rate),
    465 frames_cnt_since_drop_(0),
    466 feature_table_name_(feature_table_name)
    467 {
    468     //
    469 }
    470 
    471 VCMQMDecodeCompleCallback::~VCMQMDecodeCompleCallback()
    472  {
    473 //     if (_interpolator != NULL)
    474 //     {
    475 //         deleteInterpolator(_interpolator);
    476 //         _interpolator = NULL;
    477 //     }
    478    if (_decBuffer != NULL)
    479    {
    480      delete [] _decBuffer;
    481      _decBuffer = NULL;
    482    }
    483  }
    484 
    485 int32_t
    486 VCMQMDecodeCompleCallback::FrameToRender(I420VideoFrame& videoFrame)
    487 {
    488   ++frames_cnt_since_drop_;
    489 
    490   // When receiving the first coded frame the last_frame variable is not set
    491   if (last_frame_.IsZeroSize()) {
    492     last_frame_.CopyFrame(videoFrame);
    493   }
    494 
    495    // Check if there were frames skipped.
    496   int num_frames_skipped = static_cast<int>( 0.5f +
    497   (videoFrame.timestamp() - (last_frame_.timestamp() + (9e4 / frame_rate_))) /
    498   (9e4 / frame_rate_));
    499 
    500   // If so...put the last frames into the encoded stream to make up for the
    501   // skipped frame(s)
    502   while (num_frames_skipped > 0) {
    503     PrintI420VideoFrame(last_frame_, _decodedFile);
    504     _frameCnt++;
    505     --num_frames_skipped;
    506     frames_cnt_since_drop_ = 1; // Reset counter
    507 
    508   }
    509 
    510   DataLog::InsertCell(
    511         feature_table_name_,"num frames since drop",frames_cnt_since_drop_);
    512 
    513   if (_origWidth == videoFrame.width() && _origHeight == videoFrame.height())
    514   {
    515     if (PrintI420VideoFrame(videoFrame, _decodedFile) < 0) {
    516       return -1;
    517     }
    518     _frameCnt++;
    519     // no need for interpolator and decBuffer
    520     if (_decBuffer != NULL)
    521     {
    522       delete [] _decBuffer;
    523       _decBuffer = NULL;
    524     }
    525     _decWidth = 0;
    526     _decHeight = 0;
    527   }
    528   else
    529   {
    530     // TODO(mikhal): Add support for scaling.
    531     return -1;
    532   }
    533 
    534   _decodedBytes += CalcBufferSize(kI420, videoFrame.width(),
    535                                   videoFrame.height());
    536   videoFrame.SwapFrame(&last_frame_);
    537   return VCM_OK;
    538 }
    539 
    540 int32_t VCMQMDecodeCompleCallback::DecodedBytes()
    541 {
    542   return _decodedBytes;
    543 }
    544 
    545 void VCMQMDecodeCompleCallback::SetOriginalFrameDimensions(int32_t width,
    546                                                            int32_t height)
    547 {
    548   _origWidth = width;
    549   _origHeight = height;
    550 }
    551 
    552 int32_t VCMQMDecodeCompleCallback::buildInterpolator()
    553 {
    554   uint32_t decFrameLength  = _origWidth*_origHeight*3 >> 1;
    555   if (_decBuffer != NULL)
    556   {
    557     delete [] _decBuffer;
    558   }
    559   _decBuffer = new uint8_t[decFrameLength];
    560   if (_decBuffer == NULL)
    561   {
    562     return -1;
    563   }
    564   return 0;
    565 }
    566 
    567 // This function checks if the total number of frames processed in the encoding
    568 // process is the same as the number of frames rendered. If not,  the last
    569 // frame (or several consecutive frames from the end) must have been dropped. If
    570 // this is the case, the last frame is repeated so that there are as many
    571 // frames rendered as there are number of frames encoded.
    572 void VCMQMDecodeCompleCallback::WriteEnd(int input_frame_count)
    573 {
    574   int num_missing_frames = input_frame_count - _frameCnt;
    575 
    576   for (int n = num_missing_frames; n > 0; --n) {
    577     PrintI420VideoFrame(last_frame_, _decodedFile);
    578     _frameCnt++;
    579   }
    580 }
    581