1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #include "webrtc/common_video/libyuv/include/scaler.h" 11 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 12 #include "webrtc/modules/video_processing/video_denoiser.h" 13 14 namespace webrtc { 15 16 VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection) 17 : width_(0), 18 height_(0), 19 filter_(DenoiserFilter::Create(runtime_cpu_detection)) {} 20 21 void VideoDenoiser::TrailingReduction(int mb_rows, 22 int mb_cols, 23 const uint8_t* y_src, 24 int stride_y, 25 uint8_t* y_dst) { 26 for (int mb_row = 1; mb_row < mb_rows - 1; ++mb_row) { 27 for (int mb_col = 1; mb_col < mb_cols - 1; ++mb_col) { 28 int mb_index = mb_row * mb_cols + mb_col; 29 uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4); 30 const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4); 31 // If the number of denoised neighbors is less than a threshold, 32 // do NOT denoise for the block. Set different threshold for skin MB. 33 // The change of denoising status will not propagate. 34 if (metrics_[mb_index].is_skin) { 35 // The threshold is high (more strict) for non-skin MB where the 36 // trailing usually happen. 37 if (metrics_[mb_index].denoise && 38 metrics_[mb_index + 1].denoise + metrics_[mb_index - 1].denoise + 39 metrics_[mb_index + mb_cols].denoise + 40 metrics_[mb_index - mb_cols].denoise <= 41 2) { 42 metrics_[mb_index].denoise = 0; 43 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y); 44 } 45 } else if (metrics_[mb_index].denoise && 46 metrics_[mb_index + 1].denoise + 47 metrics_[mb_index - 1].denoise + 48 metrics_[mb_index + mb_cols + 1].denoise + 49 metrics_[mb_index + mb_cols - 1].denoise + 50 metrics_[mb_index - mb_cols + 1].denoise + 51 metrics_[mb_index - mb_cols - 1].denoise + 52 metrics_[mb_index + mb_cols].denoise + 53 metrics_[mb_index - mb_cols].denoise <= 54 7) { 55 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y); 56 } 57 } 58 } 59 } 60 61 void VideoDenoiser::DenoiseFrame(const VideoFrame& frame, 62 VideoFrame* denoised_frame) { 63 int stride_y = frame.stride(kYPlane); 64 int stride_u = frame.stride(kUPlane); 65 int stride_v = frame.stride(kVPlane); 66 // If previous width and height are different from current frame's, then no 67 // denoising for the current frame. 68 if (width_ != frame.width() || height_ != frame.height()) { 69 width_ = frame.width(); 70 height_ = frame.height(); 71 denoised_frame->CreateFrame(frame.buffer(kYPlane), frame.buffer(kUPlane), 72 frame.buffer(kVPlane), width_, height_, 73 stride_y, stride_u, stride_v); 74 // Setting time parameters to the output frame. 75 denoised_frame->set_timestamp(frame.timestamp()); 76 denoised_frame->set_render_time_ms(frame.render_time_ms()); 77 return; 78 } 79 // For 16x16 block. 80 int mb_cols = width_ >> 4; 81 int mb_rows = height_ >> 4; 82 if (metrics_.get() == nullptr) 83 metrics_.reset(new DenoiseMetrics[mb_cols * mb_rows]()); 84 // Denoise on Y plane. 85 uint8_t* y_dst = denoised_frame->buffer(kYPlane); 86 uint8_t* u_dst = denoised_frame->buffer(kUPlane); 87 uint8_t* v_dst = denoised_frame->buffer(kVPlane); 88 const uint8_t* y_src = frame.buffer(kYPlane); 89 const uint8_t* u_src = frame.buffer(kUPlane); 90 const uint8_t* v_src = frame.buffer(kVPlane); 91 // Temporary buffer to store denoising result. 92 uint8_t y_tmp[16 * 16] = {0}; 93 for (int mb_row = 0; mb_row < mb_rows; ++mb_row) { 94 for (int mb_col = 0; mb_col < mb_cols; ++mb_col) { 95 const uint8_t* mb_src = y_src + (mb_row << 4) * stride_y + (mb_col << 4); 96 uint8_t* mb_dst = y_dst + (mb_row << 4) * stride_y + (mb_col << 4); 97 int mb_index = mb_row * mb_cols + mb_col; 98 // Denoise each MB at the very start and save the result to a temporary 99 // buffer. 100 if (filter_->MbDenoise(mb_dst, stride_y, y_tmp, 16, mb_src, stride_y, 0, 101 1) == FILTER_BLOCK) { 102 uint32_t thr_var = 0; 103 // Save var and sad to the buffer. 104 metrics_[mb_index].var = filter_->Variance16x8( 105 mb_dst, stride_y, y_tmp, 16, &metrics_[mb_index].sad); 106 // Get skin map. 107 metrics_[mb_index].is_skin = MbHasSkinColor( 108 y_src, u_src, v_src, stride_y, stride_u, stride_v, mb_row, mb_col); 109 // Variance threshold for skin/non-skin MB is different. 110 // Skin MB use a small threshold to reduce blockiness. 111 thr_var = metrics_[mb_index].is_skin ? 128 : 12 * 128; 112 if (metrics_[mb_index].var > thr_var) { 113 metrics_[mb_index].denoise = 0; 114 // Use the source MB. 115 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y); 116 } else { 117 metrics_[mb_index].denoise = 1; 118 // Use the denoised MB. 119 filter_->CopyMem16x16(y_tmp, 16, mb_dst, stride_y); 120 } 121 } else { 122 metrics_[mb_index].denoise = 0; 123 filter_->CopyMem16x16(mb_src, stride_y, mb_dst, stride_y); 124 } 125 // Copy source U/V plane. 126 const uint8_t* mb_src_u = 127 u_src + (mb_row << 3) * stride_u + (mb_col << 3); 128 const uint8_t* mb_src_v = 129 v_src + (mb_row << 3) * stride_v + (mb_col << 3); 130 uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u + (mb_col << 3); 131 uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v + (mb_col << 3); 132 filter_->CopyMem8x8(mb_src_u, stride_u, mb_dst_u, stride_u); 133 filter_->CopyMem8x8(mb_src_v, stride_v, mb_dst_v, stride_v); 134 } 135 } 136 // Second round. 137 // This is to reduce the trailing artifact and blockiness by referring 138 // neighbors' denoising status. 139 TrailingReduction(mb_rows, mb_cols, y_src, stride_y, y_dst); 140 141 // Setting time parameters to the output frame. 142 denoised_frame->set_timestamp(frame.timestamp()); 143 denoised_frame->set_render_time_ms(frame.render_time_ms()); 144 return; 145 } 146 147 } // namespace webrtc 148
