1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <stddef.h> // size_t 12 #include <string> 13 #include <vector> 14 15 #include "testing/gtest/include/gtest/gtest.h" 16 #include "webrtc/audio_processing/debug.pb.h" 17 #include "webrtc/base/checks.h" 18 #include "webrtc/base/scoped_ptr.h" 19 #include "webrtc/common_audio/channel_buffer.h" 20 #include "webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h" 21 #include "webrtc/modules/audio_processing/include/audio_processing.h" 22 #include "webrtc/modules/audio_processing/test/protobuf_utils.h" 23 #include "webrtc/modules/audio_processing/test/test_utils.h" 24 #include "webrtc/test/testsupport/fileutils.h" 25 26 namespace webrtc { 27 namespace test { 28 29 namespace { 30 31 void MaybeResetBuffer(rtc::scoped_ptr<ChannelBuffer<float>>* buffer, 32 const StreamConfig& config) { 33 auto& buffer_ref = *buffer; 34 if (!buffer_ref.get() || buffer_ref->num_frames() != config.num_frames() || 35 buffer_ref->num_channels() != config.num_channels()) { 36 buffer_ref.reset(new ChannelBuffer<float>(config.num_frames(), 37 config.num_channels())); 38 } 39 } 40 41 class DebugDumpGenerator { 42 public: 43 DebugDumpGenerator(const std::string& input_file_name, 44 int input_file_rate_hz, 45 int input_channels, 46 const std::string& reverse_file_name, 47 int reverse_file_rate_hz, 48 int reverse_channels, 49 const Config& config, 50 const std::string& dump_file_name); 51 52 // Constructor that uses default input files. 53 explicit DebugDumpGenerator(const Config& config); 54 55 ~DebugDumpGenerator(); 56 57 // Changes the sample rate of the input audio to the APM. 58 void SetInputRate(int rate_hz); 59 60 // Sets if converts stereo input signal to mono by discarding other channels. 61 void ForceInputMono(bool mono); 62 63 // Changes the sample rate of the reverse audio to the APM. 64 void SetReverseRate(int rate_hz); 65 66 // Sets if converts stereo reverse signal to mono by discarding other 67 // channels. 68 void ForceReverseMono(bool mono); 69 70 // Sets the required sample rate of the APM output. 71 void SetOutputRate(int rate_hz); 72 73 // Sets the required channels of the APM output. 74 void SetOutputChannels(int channels); 75 76 std::string dump_file_name() const { return dump_file_name_; } 77 78 void StartRecording(); 79 void Process(size_t num_blocks); 80 void StopRecording(); 81 AudioProcessing* apm() const { return apm_.get(); } 82 83 private: 84 static void ReadAndDeinterleave(ResampleInputAudioFile* audio, int channels, 85 const StreamConfig& config, 86 float* const* buffer); 87 88 // APM input/output settings. 89 StreamConfig input_config_; 90 StreamConfig reverse_config_; 91 StreamConfig output_config_; 92 93 // Input file format. 94 const std::string input_file_name_; 95 ResampleInputAudioFile input_audio_; 96 const int input_file_channels_; 97 98 // Reverse file format. 99 const std::string reverse_file_name_; 100 ResampleInputAudioFile reverse_audio_; 101 const int reverse_file_channels_; 102 103 // Buffer for APM input/output. 104 rtc::scoped_ptr<ChannelBuffer<float>> input_; 105 rtc::scoped_ptr<ChannelBuffer<float>> reverse_; 106 rtc::scoped_ptr<ChannelBuffer<float>> output_; 107 108 rtc::scoped_ptr<AudioProcessing> apm_; 109 110 const std::string dump_file_name_; 111 }; 112 113 DebugDumpGenerator::DebugDumpGenerator(const std::string& input_file_name, 114 int input_rate_hz, 115 int input_channels, 116 const std::string& reverse_file_name, 117 int reverse_rate_hz, 118 int reverse_channels, 119 const Config& config, 120 const std::string& dump_file_name) 121 : input_config_(input_rate_hz, input_channels), 122 reverse_config_(reverse_rate_hz, reverse_channels), 123 output_config_(input_rate_hz, input_channels), 124 input_audio_(input_file_name, input_rate_hz, input_rate_hz), 125 input_file_channels_(input_channels), 126 reverse_audio_(reverse_file_name, reverse_rate_hz, reverse_rate_hz), 127 reverse_file_channels_(reverse_channels), 128 input_(new ChannelBuffer<float>(input_config_.num_frames(), 129 input_config_.num_channels())), 130 reverse_(new ChannelBuffer<float>(reverse_config_.num_frames(), 131 reverse_config_.num_channels())), 132 output_(new ChannelBuffer<float>(output_config_.num_frames(), 133 output_config_.num_channels())), 134 apm_(AudioProcessing::Create(config)), 135 dump_file_name_(dump_file_name) { 136 } 137 138 DebugDumpGenerator::DebugDumpGenerator(const Config& config) 139 : DebugDumpGenerator(ResourcePath("near32_stereo", "pcm"), 32000, 2, 140 ResourcePath("far32_stereo", "pcm"), 32000, 2, 141 config, 142 TempFilename(OutputPath(), "debug_aec")) { 143 } 144 145 DebugDumpGenerator::~DebugDumpGenerator() { 146 remove(dump_file_name_.c_str()); 147 } 148 149 void DebugDumpGenerator::SetInputRate(int rate_hz) { 150 input_audio_.set_output_rate_hz(rate_hz); 151 input_config_.set_sample_rate_hz(rate_hz); 152 MaybeResetBuffer(&input_, input_config_); 153 } 154 155 void DebugDumpGenerator::ForceInputMono(bool mono) { 156 const int channels = mono ? 1 : input_file_channels_; 157 input_config_.set_num_channels(channels); 158 MaybeResetBuffer(&input_, input_config_); 159 } 160 161 void DebugDumpGenerator::SetReverseRate(int rate_hz) { 162 reverse_audio_.set_output_rate_hz(rate_hz); 163 reverse_config_.set_sample_rate_hz(rate_hz); 164 MaybeResetBuffer(&reverse_, reverse_config_); 165 } 166 167 void DebugDumpGenerator::ForceReverseMono(bool mono) { 168 const int channels = mono ? 1 : reverse_file_channels_; 169 reverse_config_.set_num_channels(channels); 170 MaybeResetBuffer(&reverse_, reverse_config_); 171 } 172 173 void DebugDumpGenerator::SetOutputRate(int rate_hz) { 174 output_config_.set_sample_rate_hz(rate_hz); 175 MaybeResetBuffer(&output_, output_config_); 176 } 177 178 void DebugDumpGenerator::SetOutputChannels(int channels) { 179 output_config_.set_num_channels(channels); 180 MaybeResetBuffer(&output_, output_config_); 181 } 182 183 void DebugDumpGenerator::StartRecording() { 184 apm_->StartDebugRecording(dump_file_name_.c_str()); 185 } 186 187 void DebugDumpGenerator::Process(size_t num_blocks) { 188 for (size_t i = 0; i < num_blocks; ++i) { 189 ReadAndDeinterleave(&reverse_audio_, reverse_file_channels_, 190 reverse_config_, reverse_->channels()); 191 ReadAndDeinterleave(&input_audio_, input_file_channels_, input_config_, 192 input_->channels()); 193 RTC_CHECK_EQ(AudioProcessing::kNoError, apm_->set_stream_delay_ms(100)); 194 apm_->set_stream_key_pressed(i % 10 == 9); 195 RTC_CHECK_EQ(AudioProcessing::kNoError, 196 apm_->ProcessStream(input_->channels(), input_config_, 197 output_config_, output_->channels())); 198 199 RTC_CHECK_EQ(AudioProcessing::kNoError, 200 apm_->ProcessReverseStream(reverse_->channels(), 201 reverse_config_, 202 reverse_config_, 203 reverse_->channels())); 204 } 205 } 206 207 void DebugDumpGenerator::StopRecording() { 208 apm_->StopDebugRecording(); 209 } 210 211 void DebugDumpGenerator::ReadAndDeinterleave(ResampleInputAudioFile* audio, 212 int channels, 213 const StreamConfig& config, 214 float* const* buffer) { 215 const size_t num_frames = config.num_frames(); 216 const int out_channels = config.num_channels(); 217 218 std::vector<int16_t> signal(channels * num_frames); 219 220 audio->Read(num_frames * channels, &signal[0]); 221 222 // We only allow reducing number of channels by discarding some channels. 223 RTC_CHECK_LE(out_channels, channels); 224 for (int channel = 0; channel < out_channels; ++channel) { 225 for (size_t i = 0; i < num_frames; ++i) { 226 buffer[channel][i] = S16ToFloat(signal[i * channels + channel]); 227 } 228 } 229 } 230 231 } // namespace 232 233 class DebugDumpTest : public ::testing::Test { 234 public: 235 DebugDumpTest(); 236 237 // VerifyDebugDump replays a debug dump using APM and verifies that the result 238 // is bit-exact-identical to the output channel in the dump. This is only 239 // guaranteed if the debug dump is started on the first frame. 240 void VerifyDebugDump(const std::string& dump_file_name); 241 242 private: 243 // Following functions are facilities for replaying debug dumps. 244 void OnInitEvent(const audioproc::Init& msg); 245 void OnStreamEvent(const audioproc::Stream& msg); 246 void OnReverseStreamEvent(const audioproc::ReverseStream& msg); 247 void OnConfigEvent(const audioproc::Config& msg); 248 249 void MaybeRecreateApm(const audioproc::Config& msg); 250 void ConfigureApm(const audioproc::Config& msg); 251 252 // Buffer for APM input/output. 253 rtc::scoped_ptr<ChannelBuffer<float>> input_; 254 rtc::scoped_ptr<ChannelBuffer<float>> reverse_; 255 rtc::scoped_ptr<ChannelBuffer<float>> output_; 256 257 rtc::scoped_ptr<AudioProcessing> apm_; 258 259 StreamConfig input_config_; 260 StreamConfig reverse_config_; 261 StreamConfig output_config_; 262 }; 263 264 DebugDumpTest::DebugDumpTest() 265 : input_(nullptr), // will be created upon usage. 266 reverse_(nullptr), 267 output_(nullptr), 268 apm_(nullptr) { 269 } 270 271 void DebugDumpTest::VerifyDebugDump(const std::string& in_filename) { 272 FILE* in_file = fopen(in_filename.c_str(), "rb"); 273 ASSERT_TRUE(in_file); 274 audioproc::Event event_msg; 275 276 while (ReadMessageFromFile(in_file, &event_msg)) { 277 switch (event_msg.type()) { 278 case audioproc::Event::INIT: 279 OnInitEvent(event_msg.init()); 280 break; 281 case audioproc::Event::STREAM: 282 OnStreamEvent(event_msg.stream()); 283 break; 284 case audioproc::Event::REVERSE_STREAM: 285 OnReverseStreamEvent(event_msg.reverse_stream()); 286 break; 287 case audioproc::Event::CONFIG: 288 OnConfigEvent(event_msg.config()); 289 break; 290 case audioproc::Event::UNKNOWN_EVENT: 291 // We do not expect receive UNKNOWN event currently. 292 FAIL(); 293 } 294 } 295 fclose(in_file); 296 } 297 298 // OnInitEvent reset the input/output/reserve channel format. 299 void DebugDumpTest::OnInitEvent(const audioproc::Init& msg) { 300 ASSERT_TRUE(msg.has_num_input_channels()); 301 ASSERT_TRUE(msg.has_output_sample_rate()); 302 ASSERT_TRUE(msg.has_num_output_channels()); 303 ASSERT_TRUE(msg.has_reverse_sample_rate()); 304 ASSERT_TRUE(msg.has_num_reverse_channels()); 305 306 input_config_ = StreamConfig(msg.sample_rate(), msg.num_input_channels()); 307 output_config_ = 308 StreamConfig(msg.output_sample_rate(), msg.num_output_channels()); 309 reverse_config_ = 310 StreamConfig(msg.reverse_sample_rate(), msg.num_reverse_channels()); 311 312 MaybeResetBuffer(&input_, input_config_); 313 MaybeResetBuffer(&output_, output_config_); 314 MaybeResetBuffer(&reverse_, reverse_config_); 315 } 316 317 // OnStreamEvent replays an input signal and verifies the output. 318 void DebugDumpTest::OnStreamEvent(const audioproc::Stream& msg) { 319 // APM should have been created. 320 ASSERT_TRUE(apm_.get()); 321 322 EXPECT_NOERR(apm_->gain_control()->set_stream_analog_level(msg.level())); 323 EXPECT_NOERR(apm_->set_stream_delay_ms(msg.delay())); 324 apm_->echo_cancellation()->set_stream_drift_samples(msg.drift()); 325 if (msg.has_keypress()) 326 apm_->set_stream_key_pressed(msg.keypress()); 327 else 328 apm_->set_stream_key_pressed(true); 329 330 ASSERT_EQ(input_config_.num_channels(), 331 static_cast<size_t>(msg.input_channel_size())); 332 ASSERT_EQ(input_config_.num_frames() * sizeof(float), 333 msg.input_channel(0).size()); 334 335 for (int i = 0; i < msg.input_channel_size(); ++i) { 336 memcpy(input_->channels()[i], msg.input_channel(i).data(), 337 msg.input_channel(i).size()); 338 } 339 340 ASSERT_EQ(AudioProcessing::kNoError, 341 apm_->ProcessStream(input_->channels(), input_config_, 342 output_config_, output_->channels())); 343 344 // Check that output of APM is bit-exact to the output in the dump. 345 ASSERT_EQ(output_config_.num_channels(), 346 static_cast<size_t>(msg.output_channel_size())); 347 ASSERT_EQ(output_config_.num_frames() * sizeof(float), 348 msg.output_channel(0).size()); 349 for (int i = 0; i < msg.output_channel_size(); ++i) { 350 ASSERT_EQ(0, memcmp(output_->channels()[i], msg.output_channel(i).data(), 351 msg.output_channel(i).size())); 352 } 353 } 354 355 void DebugDumpTest::OnReverseStreamEvent(const audioproc::ReverseStream& msg) { 356 // APM should have been created. 357 ASSERT_TRUE(apm_.get()); 358 359 ASSERT_GT(msg.channel_size(), 0); 360 ASSERT_EQ(reverse_config_.num_channels(), 361 static_cast<size_t>(msg.channel_size())); 362 ASSERT_EQ(reverse_config_.num_frames() * sizeof(float), 363 msg.channel(0).size()); 364 365 for (int i = 0; i < msg.channel_size(); ++i) { 366 memcpy(reverse_->channels()[i], msg.channel(i).data(), 367 msg.channel(i).size()); 368 } 369 370 ASSERT_EQ(AudioProcessing::kNoError, 371 apm_->ProcessReverseStream(reverse_->channels(), 372 reverse_config_, 373 reverse_config_, 374 reverse_->channels())); 375 } 376 377 void DebugDumpTest::OnConfigEvent(const audioproc::Config& msg) { 378 MaybeRecreateApm(msg); 379 ConfigureApm(msg); 380 } 381 382 void DebugDumpTest::MaybeRecreateApm(const audioproc::Config& msg) { 383 // These configurations cannot be changed on the fly. 384 Config config; 385 ASSERT_TRUE(msg.has_aec_delay_agnostic_enabled()); 386 config.Set<DelayAgnostic>( 387 new DelayAgnostic(msg.aec_delay_agnostic_enabled())); 388 389 ASSERT_TRUE(msg.has_noise_robust_agc_enabled()); 390 config.Set<ExperimentalAgc>( 391 new ExperimentalAgc(msg.noise_robust_agc_enabled())); 392 393 ASSERT_TRUE(msg.has_transient_suppression_enabled()); 394 config.Set<ExperimentalNs>( 395 new ExperimentalNs(msg.transient_suppression_enabled())); 396 397 ASSERT_TRUE(msg.has_aec_extended_filter_enabled()); 398 config.Set<ExtendedFilter>(new ExtendedFilter( 399 msg.aec_extended_filter_enabled())); 400 401 // We only create APM once, since changes on these fields should not 402 // happen in current implementation. 403 if (!apm_.get()) { 404 apm_.reset(AudioProcessing::Create(config)); 405 } 406 } 407 408 void DebugDumpTest::ConfigureApm(const audioproc::Config& msg) { 409 // AEC configs. 410 ASSERT_TRUE(msg.has_aec_enabled()); 411 EXPECT_EQ(AudioProcessing::kNoError, 412 apm_->echo_cancellation()->Enable(msg.aec_enabled())); 413 414 ASSERT_TRUE(msg.has_aec_drift_compensation_enabled()); 415 EXPECT_EQ(AudioProcessing::kNoError, 416 apm_->echo_cancellation()->enable_drift_compensation( 417 msg.aec_drift_compensation_enabled())); 418 419 ASSERT_TRUE(msg.has_aec_suppression_level()); 420 EXPECT_EQ(AudioProcessing::kNoError, 421 apm_->echo_cancellation()->set_suppression_level( 422 static_cast<EchoCancellation::SuppressionLevel>( 423 msg.aec_suppression_level()))); 424 425 // AECM configs. 426 ASSERT_TRUE(msg.has_aecm_enabled()); 427 EXPECT_EQ(AudioProcessing::kNoError, 428 apm_->echo_control_mobile()->Enable(msg.aecm_enabled())); 429 430 ASSERT_TRUE(msg.has_aecm_comfort_noise_enabled()); 431 EXPECT_EQ(AudioProcessing::kNoError, 432 apm_->echo_control_mobile()->enable_comfort_noise( 433 msg.aecm_comfort_noise_enabled())); 434 435 ASSERT_TRUE(msg.has_aecm_routing_mode()); 436 EXPECT_EQ(AudioProcessing::kNoError, 437 apm_->echo_control_mobile()->set_routing_mode( 438 static_cast<EchoControlMobile::RoutingMode>( 439 msg.aecm_routing_mode()))); 440 441 // AGC configs. 442 ASSERT_TRUE(msg.has_agc_enabled()); 443 EXPECT_EQ(AudioProcessing::kNoError, 444 apm_->gain_control()->Enable(msg.agc_enabled())); 445 446 ASSERT_TRUE(msg.has_agc_mode()); 447 EXPECT_EQ(AudioProcessing::kNoError, 448 apm_->gain_control()->set_mode( 449 static_cast<GainControl::Mode>(msg.agc_mode()))); 450 451 ASSERT_TRUE(msg.has_agc_limiter_enabled()); 452 EXPECT_EQ(AudioProcessing::kNoError, 453 apm_->gain_control()->enable_limiter(msg.agc_limiter_enabled())); 454 455 // HPF configs. 456 ASSERT_TRUE(msg.has_hpf_enabled()); 457 EXPECT_EQ(AudioProcessing::kNoError, 458 apm_->high_pass_filter()->Enable(msg.hpf_enabled())); 459 460 // NS configs. 461 ASSERT_TRUE(msg.has_ns_enabled()); 462 EXPECT_EQ(AudioProcessing::kNoError, 463 apm_->noise_suppression()->Enable(msg.ns_enabled())); 464 465 ASSERT_TRUE(msg.has_ns_level()); 466 EXPECT_EQ(AudioProcessing::kNoError, 467 apm_->noise_suppression()->set_level( 468 static_cast<NoiseSuppression::Level>(msg.ns_level()))); 469 } 470 471 TEST_F(DebugDumpTest, SimpleCase) { 472 Config config; 473 DebugDumpGenerator generator(config); 474 generator.StartRecording(); 475 generator.Process(100); 476 generator.StopRecording(); 477 VerifyDebugDump(generator.dump_file_name()); 478 } 479 480 TEST_F(DebugDumpTest, ChangeInputFormat) { 481 Config config; 482 DebugDumpGenerator generator(config); 483 generator.StartRecording(); 484 generator.Process(100); 485 generator.SetInputRate(48000); 486 487 generator.ForceInputMono(true); 488 // Number of output channel should not be larger than that of input. APM will 489 // fail otherwise. 490 generator.SetOutputChannels(1); 491 492 generator.Process(100); 493 generator.StopRecording(); 494 VerifyDebugDump(generator.dump_file_name()); 495 } 496 497 TEST_F(DebugDumpTest, ChangeReverseFormat) { 498 Config config; 499 DebugDumpGenerator generator(config); 500 generator.StartRecording(); 501 generator.Process(100); 502 generator.SetReverseRate(48000); 503 generator.ForceReverseMono(true); 504 generator.Process(100); 505 generator.StopRecording(); 506 VerifyDebugDump(generator.dump_file_name()); 507 } 508 509 TEST_F(DebugDumpTest, ChangeOutputFormat) { 510 Config config; 511 DebugDumpGenerator generator(config); 512 generator.StartRecording(); 513 generator.Process(100); 514 generator.SetOutputRate(48000); 515 generator.SetOutputChannels(1); 516 generator.Process(100); 517 generator.StopRecording(); 518 VerifyDebugDump(generator.dump_file_name()); 519 } 520 521 TEST_F(DebugDumpTest, ToggleAec) { 522 Config config; 523 DebugDumpGenerator generator(config); 524 generator.StartRecording(); 525 generator.Process(100); 526 527 EchoCancellation* aec = generator.apm()->echo_cancellation(); 528 EXPECT_EQ(AudioProcessing::kNoError, aec->Enable(!aec->is_enabled())); 529 530 generator.Process(100); 531 generator.StopRecording(); 532 VerifyDebugDump(generator.dump_file_name()); 533 } 534 535 TEST_F(DebugDumpTest, ToggleDelayAgnosticAec) { 536 Config config; 537 config.Set<DelayAgnostic>(new DelayAgnostic(true)); 538 DebugDumpGenerator generator(config); 539 generator.StartRecording(); 540 generator.Process(100); 541 542 EchoCancellation* aec = generator.apm()->echo_cancellation(); 543 EXPECT_EQ(AudioProcessing::kNoError, aec->Enable(!aec->is_enabled())); 544 545 generator.Process(100); 546 generator.StopRecording(); 547 VerifyDebugDump(generator.dump_file_name()); 548 } 549 550 TEST_F(DebugDumpTest, ToggleAecLevel) { 551 Config config; 552 DebugDumpGenerator generator(config); 553 EchoCancellation* aec = generator.apm()->echo_cancellation(); 554 EXPECT_EQ(AudioProcessing::kNoError, aec->Enable(true)); 555 EXPECT_EQ(AudioProcessing::kNoError, 556 aec->set_suppression_level(EchoCancellation::kLowSuppression)); 557 generator.StartRecording(); 558 generator.Process(100); 559 560 EXPECT_EQ(AudioProcessing::kNoError, 561 aec->set_suppression_level(EchoCancellation::kHighSuppression)); 562 generator.Process(100); 563 generator.StopRecording(); 564 VerifyDebugDump(generator.dump_file_name()); 565 } 566 567 #if defined(WEBRTC_ANDROID) 568 // AGC may not be supported on Android. 569 #define MAYBE_ToggleAgc DISABLED_ToggleAgc 570 #else 571 #define MAYBE_ToggleAgc ToggleAgc 572 #endif 573 TEST_F(DebugDumpTest, MAYBE_ToggleAgc) { 574 Config config; 575 DebugDumpGenerator generator(config); 576 generator.StartRecording(); 577 generator.Process(100); 578 579 GainControl* agc = generator.apm()->gain_control(); 580 EXPECT_EQ(AudioProcessing::kNoError, agc->Enable(!agc->is_enabled())); 581 582 generator.Process(100); 583 generator.StopRecording(); 584 VerifyDebugDump(generator.dump_file_name()); 585 } 586 587 TEST_F(DebugDumpTest, ToggleNs) { 588 Config config; 589 DebugDumpGenerator generator(config); 590 generator.StartRecording(); 591 generator.Process(100); 592 593 NoiseSuppression* ns = generator.apm()->noise_suppression(); 594 EXPECT_EQ(AudioProcessing::kNoError, ns->Enable(!ns->is_enabled())); 595 596 generator.Process(100); 597 generator.StopRecording(); 598 VerifyDebugDump(generator.dump_file_name()); 599 } 600 601 TEST_F(DebugDumpTest, TransientSuppressionOn) { 602 Config config; 603 config.Set<ExperimentalNs>(new ExperimentalNs(true)); 604 DebugDumpGenerator generator(config); 605 generator.StartRecording(); 606 generator.Process(100); 607 generator.StopRecording(); 608 VerifyDebugDump(generator.dump_file_name()); 609 } 610 611 } // namespace test 612 } // namespace webrtc 613