1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/modules/video_coding/codecs/test_framework/benchmark.h" 12 13 #include <assert.h> 14 15 #include <iostream> 16 #include <sstream> 17 #include <vector> 18 #if defined(_WIN32) 19 #include <windows.h> 20 #endif 21 22 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 23 #include "webrtc/modules/video_coding/codecs/test_framework/video_source.h" 24 #include "webrtc/system_wrappers/interface/event_wrapper.h" 25 #include "webrtc/test/testsupport/fileutils.h" 26 #include "webrtc/test/testsupport/metrics/video_metrics.h" 27 28 #define SSIM_CALC 0 // by default, don't compute SSIM 29 30 using namespace webrtc; 31 32 Benchmark::Benchmark() 33 : 34 NormalAsyncTest("Benchmark", "Codec benchmark over a range of test cases", 6), 35 _resultsFileName(webrtc::test::OutputPath() + "benchmark.txt"), 36 _codecName("Default") 37 { 38 } 39 40 Benchmark::Benchmark(std::string name, std::string description) 41 : 42 NormalAsyncTest(name, description, 6), 43 _resultsFileName(webrtc::test::OutputPath() + "benchmark.txt"), 44 _codecName("Default") 45 { 46 } 47 48 Benchmark::Benchmark(std::string name, std::string description, std::string resultsFileName, std::string codecName) 49 : 50 NormalAsyncTest(name, description, 6), 51 _resultsFileName(resultsFileName), 52 _codecName(codecName) 53 { 54 } 55 56 void 57 Benchmark::Perform() 58 { 59 std::vector<const VideoSource*> sources; 60 std::vector<const VideoSource*>::iterator it; 61 62 // Configuration -------------------------- 63 sources.push_back(new const VideoSource(webrtc::test::ProjectRootPath() + 64 "resources/foreman_cif.yuv", kCIF)); 65 // sources.push_back(new const VideoSource(webrtc::test::ProjectRootPath() + 66 // "resources/akiyo_cif.yuv", kCIF)); 67 68 const VideoSize size[] = {kQCIF, kCIF}; 69 const int frameRate[] = {10, 15, 30}; 70 // Specifies the framerates for which to perform a speed test. 71 const bool speedTestMask[] = {false, false, false}; 72 const int bitRate[] = {50, 100, 200, 300, 400, 500, 600, 1000}; 73 // Determines the number of iterations to perform to arrive at the speed result. 74 enum { kSpeedTestIterations = 10 }; 75 // ---------------------------------------- 76 77 const int nFrameRates = sizeof(frameRate)/sizeof(*frameRate); 78 assert(sizeof(speedTestMask)/sizeof(*speedTestMask) == nFrameRates); 79 const int nBitrates = sizeof(bitRate)/sizeof(*bitRate); 80 int testIterations = 10; 81 82 webrtc::test::QualityMetricsResult psnr[nBitrates]; 83 webrtc::test::QualityMetricsResult ssim[nBitrates]; 84 double fps[nBitrates]; 85 double totalEncodeTime[nBitrates]; 86 double totalDecodeTime[nBitrates]; 87 88 _results.open(_resultsFileName.c_str(), std::fstream::out); 89 _results << GetMagicStr() << std::endl; 90 _results << _codecName << std::endl; 91 92 for (it = sources.begin() ; it < sources.end(); it++) 93 { 94 for (int i = 0; i < static_cast<int>(sizeof(size)/sizeof(*size)); i++) 95 { 96 for (int j = 0; j < nFrameRates; j++) 97 { 98 std::stringstream ss; 99 std::string strFrameRate; 100 std::string outFileName; 101 ss << frameRate[j]; 102 ss >> strFrameRate; 103 outFileName = (*it)->GetFilePath() + "/" + (*it)->GetName() + "_" + 104 VideoSource::GetSizeString(size[i]) + "_" + strFrameRate + ".yuv"; 105 106 _target = new const VideoSource(outFileName, size[i], frameRate[j]); 107 (*it)->Convert(*_target); 108 if (VideoSource::FileExists(outFileName.c_str())) 109 { 110 _inname = outFileName; 111 } 112 else 113 { 114 _inname = (*it)->GetFileName(); 115 } 116 117 std::cout << (*it)->GetName() << ", " << VideoSource::GetSizeString(size[i]) 118 << ", " << frameRate[j] << " fps" << std::endl << "Bitrate [kbps]:"; 119 _results << (*it)->GetName() << "," << VideoSource::GetSizeString(size[i]) 120 << "," << frameRate[j] << " fps" << std::endl << "Bitrate [kbps]"; 121 122 if (speedTestMask[j]) 123 { 124 testIterations = kSpeedTestIterations; 125 } 126 else 127 { 128 testIterations = 1; 129 } 130 131 for (int k = 0; k < nBitrates; k++) 132 { 133 _bitRate = (bitRate[k]); 134 double avgFps = 0.0; 135 totalEncodeTime[k] = 0; 136 totalDecodeTime[k] = 0; 137 138 for (int l = 0; l < testIterations; l++) 139 { 140 PerformNormalTest(); 141 _appendNext = false; 142 143 avgFps += _framecnt / (_totalEncodeTime + _totalDecodeTime); 144 totalEncodeTime[k] += _totalEncodeTime; 145 totalDecodeTime[k] += _totalDecodeTime; 146 147 } 148 avgFps /= testIterations; 149 totalEncodeTime[k] /= testIterations; 150 totalDecodeTime[k] /= testIterations; 151 152 double actualBitRate = ActualBitRate(_framecnt) / 1000.0; 153 std::cout << " " << actualBitRate; 154 _results << "," << actualBitRate; 155 webrtc::test::QualityMetricsResult psnr_result; 156 I420PSNRFromFiles(_inname.c_str(), _outname.c_str(), 157 _inst.width, _inst.height, &psnr[k]); 158 if (SSIM_CALC) 159 { 160 webrtc::test::QualityMetricsResult ssim_result; 161 I420SSIMFromFiles(_inname.c_str(), _outname.c_str(), 162 _inst.width, _inst.height, &ssim[k]); 163 164 } 165 fps[k] = avgFps; 166 } 167 std::cout << std::endl << "Y-PSNR [dB]:"; 168 _results << std::endl << "Y-PSNR [dB]"; 169 for (int k = 0; k < nBitrates; k++) 170 { 171 std::cout << " " << psnr[k].average; 172 _results << "," << psnr[k].average; 173 174 } 175 if (SSIM_CALC) 176 { 177 std::cout << std::endl << "SSIM: "; 178 _results << std::endl << "SSIM "; 179 for (int k = 0; k < nBitrates; k++) 180 { 181 std::cout << " " << ssim[k].average; 182 _results << "," << ssim[k].average; 183 } 184 185 } 186 187 std::cout << std::endl << "Encode Time[ms]:"; 188 _results << std::endl << "Encode Time[ms]"; 189 for (int k = 0; k < nBitrates; k++) 190 { 191 std::cout << " " << totalEncodeTime[k]; 192 _results << "," << totalEncodeTime[k]; 193 194 } 195 196 std::cout << std::endl << "Decode Time[ms]:"; 197 _results << std::endl << "Decode Time[ms]"; 198 for (int k = 0; k < nBitrates; k++) 199 { 200 std::cout << " " << totalDecodeTime[k]; 201 _results << "," << totalDecodeTime[k]; 202 203 } 204 205 if (speedTestMask[j]) 206 { 207 std::cout << std::endl << "Speed [fps]:"; 208 _results << std::endl << "Speed [fps]"; 209 for (int k = 0; k < nBitrates; k++) 210 { 211 std::cout << " " << static_cast<int>(fps[k] + 0.5); 212 _results << "," << static_cast<int>(fps[k] + 0.5); 213 } 214 } 215 std::cout << std::endl << std::endl; 216 _results << std::endl << std::endl; 217 218 delete _target; 219 } 220 } 221 delete *it; 222 } 223 _results.close(); 224 } 225 226 void 227 Benchmark::PerformNormalTest() 228 { 229 _encoder = GetNewEncoder(); 230 _decoder = GetNewDecoder(); 231 CodecSettings(_target->GetWidth(), _target->GetHeight(), _target->GetFrameRate(), _bitRate); 232 Setup(); 233 EventWrapper* waitEvent = EventWrapper::Create(); 234 _encoder->InitEncode(&_inst, 4, 1440); 235 CodecSpecific_InitBitrate(); 236 _decoder->InitDecode(&_inst,1); 237 238 FrameQueue frameQueue; 239 VideoEncodeCompleteCallback encCallback(_encodedFile, &frameQueue, *this); 240 VideoDecodeCompleteCallback decCallback(_decodedFile, *this); 241 _encoder->RegisterEncodeCompleteCallback(&encCallback); 242 _decoder->RegisterDecodeCompleteCallback(&decCallback); 243 244 SetCodecSpecificParameters(); 245 246 _totalEncodeTime = _totalDecodeTime = 0; 247 _totalEncodePipeTime = _totalDecodePipeTime = 0; 248 bool complete = false; 249 _framecnt = 0; 250 _encFrameCnt = 0; 251 _sumEncBytes = 0; 252 _lengthEncFrame = 0; 253 while (!complete) 254 { 255 complete = Encode(); 256 if (!frameQueue.Empty() || complete) 257 { 258 while (!frameQueue.Empty()) 259 { 260 _frameToDecode = static_cast<FrameQueueTuple *>(frameQueue.PopFrame()); 261 DoPacketLoss(); 262 int ret = Decode(); 263 delete _frameToDecode; 264 _frameToDecode = NULL; 265 if (ret < 0) 266 { 267 fprintf(stderr,"\n\nError in decoder: %d\n\n", ret); 268 exit(EXIT_FAILURE); 269 } 270 else if (ret == 0) 271 { 272 _framecnt++; 273 } 274 else 275 { 276 fprintf(stderr, "\n\nPositive return value from decode!\n\n"); 277 } 278 } 279 } 280 waitEvent->Wait(5); 281 } 282 283 _encodedVideoBuffer.Free(); 284 285 _encoder->Release(); 286 _decoder->Release(); 287 delete waitEvent; 288 delete _encoder; 289 delete _decoder; 290 Teardown(); 291 } 292 293 void 294 Benchmark::CodecSpecific_InitBitrate() 295 { 296 if (_bitRate == 0) 297 { 298 _encoder->SetRates(600, _inst.maxFramerate); 299 } 300 else 301 { 302 _encoder->SetRates(_bitRate, _inst.maxFramerate); 303 } 304 } 305
