1 /* 2 * libjingle 3 * Copyright 2011 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include "talk/media/base/videoframe.h" 29 30 #include <string.h> 31 32 #include "libyuv/compare.h" 33 #include "libyuv/planar_functions.h" 34 #include "libyuv/scale.h" 35 #include "talk/media/base/videocommon.h" 36 #include "webrtc/base/arraysize.h" 37 #include "webrtc/base/checks.h" 38 #include "webrtc/base/logging.h" 39 40 namespace cricket { 41 42 // Round to 2 pixels because Chroma channels are half size. 43 #define ROUNDTO2(v) (v & ~1) 44 45 rtc::StreamResult VideoFrame::Write(rtc::StreamInterface* stream, 46 int* error) const { 47 rtc::StreamResult result = rtc::SR_SUCCESS; 48 const uint8_t* src_y = GetYPlane(); 49 const uint8_t* src_u = GetUPlane(); 50 const uint8_t* src_v = GetVPlane(); 51 if (!src_y || !src_u || !src_v) { 52 return result; // Nothing to write. 53 } 54 const int32_t y_pitch = GetYPitch(); 55 const int32_t u_pitch = GetUPitch(); 56 const int32_t v_pitch = GetVPitch(); 57 const size_t width = GetWidth(); 58 const size_t height = GetHeight(); 59 const size_t half_width = (width + 1) >> 1; 60 const size_t half_height = (height + 1) >> 1; 61 // Write Y. 62 for (size_t row = 0; row < height; ++row) { 63 result = stream->Write(src_y + row * y_pitch, width, NULL, error); 64 if (result != rtc::SR_SUCCESS) { 65 return result; 66 } 67 } 68 // Write U. 69 for (size_t row = 0; row < half_height; ++row) { 70 result = stream->Write(src_u + row * u_pitch, half_width, NULL, error); 71 if (result != rtc::SR_SUCCESS) { 72 return result; 73 } 74 } 75 // Write V. 76 for (size_t row = 0; row < half_height; ++row) { 77 result = stream->Write(src_v + row * v_pitch, half_width, NULL, error); 78 if (result != rtc::SR_SUCCESS) { 79 return result; 80 } 81 } 82 return result; 83 } 84 85 size_t VideoFrame::CopyToBuffer(uint8_t* buffer, size_t size) const { 86 const size_t y_size = GetHeight() * GetYPitch(); 87 const size_t u_size = GetUPitch() * GetChromaHeight(); 88 const size_t v_size = GetVPitch() * GetChromaHeight(); 89 const size_t needed = y_size + u_size + v_size; 90 if (size < needed) 91 return needed; 92 CopyToPlanes(buffer, buffer + y_size, buffer + y_size + u_size, 93 GetYPitch(), GetUPitch(), GetVPitch()); 94 return needed; 95 } 96 97 bool VideoFrame::CopyToPlanes(uint8_t* dst_y, 98 uint8_t* dst_u, 99 uint8_t* dst_v, 100 int32_t dst_pitch_y, 101 int32_t dst_pitch_u, 102 int32_t dst_pitch_v) const { 103 if (!GetYPlane() || !GetUPlane() || !GetVPlane()) { 104 LOG(LS_ERROR) << "NULL plane pointer."; 105 return false; 106 } 107 int32_t src_width = static_cast<int>(GetWidth()); 108 int32_t src_height = static_cast<int>(GetHeight()); 109 return libyuv::I420Copy(GetYPlane(), GetYPitch(), 110 GetUPlane(), GetUPitch(), 111 GetVPlane(), GetVPitch(), 112 dst_y, dst_pitch_y, 113 dst_u, dst_pitch_u, 114 dst_v, dst_pitch_v, 115 src_width, src_height) == 0; 116 } 117 118 void VideoFrame::CopyToFrame(VideoFrame* dst) const { 119 if (!dst) { 120 LOG(LS_ERROR) << "NULL dst pointer."; 121 return; 122 } 123 124 CopyToPlanes(dst->GetYPlane(), dst->GetUPlane(), dst->GetVPlane(), 125 dst->GetYPitch(), dst->GetUPitch(), dst->GetVPitch()); 126 } 127 128 size_t VideoFrame::ConvertToRgbBuffer(uint32_t to_fourcc, 129 uint8_t* buffer, 130 size_t size, 131 int stride_rgb) const { 132 const size_t needed = std::abs(stride_rgb) * GetHeight(); 133 if (size < needed) { 134 LOG(LS_WARNING) << "RGB buffer is not large enough"; 135 return needed; 136 } 137 138 if (libyuv::ConvertFromI420(GetYPlane(), GetYPitch(), GetUPlane(), 139 GetUPitch(), GetVPlane(), GetVPitch(), buffer, 140 stride_rgb, static_cast<int>(GetWidth()), 141 static_cast<int>(GetHeight()), to_fourcc)) { 142 LOG(LS_ERROR) << "RGB type not supported: " << to_fourcc; 143 return 0; // 0 indicates error 144 } 145 return needed; 146 } 147 148 // TODO(fbarchard): Handle odd width/height with rounding. 149 void VideoFrame::StretchToPlanes(uint8_t* dst_y, 150 uint8_t* dst_u, 151 uint8_t* dst_v, 152 int32_t dst_pitch_y, 153 int32_t dst_pitch_u, 154 int32_t dst_pitch_v, 155 size_t width, 156 size_t height, 157 bool interpolate, 158 bool vert_crop) const { 159 if (!GetYPlane() || !GetUPlane() || !GetVPlane()) { 160 LOG(LS_ERROR) << "NULL plane pointer."; 161 return; 162 } 163 164 size_t src_width = GetWidth(); 165 size_t src_height = GetHeight(); 166 if (width == src_width && height == src_height) { 167 CopyToPlanes(dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v); 168 return; 169 } 170 const uint8_t* src_y = GetYPlane(); 171 const uint8_t* src_u = GetUPlane(); 172 const uint8_t* src_v = GetVPlane(); 173 174 if (vert_crop) { 175 // Adjust the input width:height ratio to be the same as the output ratio. 176 if (src_width * height > src_height * width) { 177 // Reduce the input width, but keep size/position aligned for YuvScaler 178 src_width = ROUNDTO2(src_height * width / height); 179 int32_t iwidth_offset = ROUNDTO2((GetWidth() - src_width) / 2); 180 src_y += iwidth_offset; 181 src_u += iwidth_offset / 2; 182 src_v += iwidth_offset / 2; 183 } else if (src_width * height < src_height * width) { 184 // Reduce the input height. 185 src_height = src_width * height / width; 186 int32_t iheight_offset = 187 static_cast<int32_t>((GetHeight() - src_height) >> 2); 188 iheight_offset <<= 1; // Ensure that iheight_offset is even. 189 src_y += iheight_offset * GetYPitch(); 190 src_u += iheight_offset / 2 * GetUPitch(); 191 src_v += iheight_offset / 2 * GetVPitch(); 192 } 193 } 194 195 // Scale to the output I420 frame. 196 libyuv::Scale(src_y, src_u, src_v, 197 GetYPitch(), GetUPitch(), GetVPitch(), 198 static_cast<int>(src_width), static_cast<int>(src_height), 199 dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v, 200 static_cast<int>(width), static_cast<int>(height), interpolate); 201 } 202 203 void VideoFrame::StretchToFrame(VideoFrame* dst, 204 bool interpolate, bool vert_crop) const { 205 if (!dst) { 206 LOG(LS_ERROR) << "NULL dst pointer."; 207 return; 208 } 209 210 StretchToPlanes(dst->GetYPlane(), dst->GetUPlane(), dst->GetVPlane(), 211 dst->GetYPitch(), dst->GetUPitch(), dst->GetVPitch(), 212 dst->GetWidth(), dst->GetHeight(), 213 interpolate, vert_crop); 214 dst->SetTimeStamp(GetTimeStamp()); 215 // Stretched frame should have the same rotation as the source. 216 dst->SetRotation(GetVideoRotation()); 217 } 218 219 VideoFrame* VideoFrame::Stretch(size_t dst_width, size_t dst_height, 220 bool interpolate, bool vert_crop) const { 221 VideoFrame* dest = CreateEmptyFrame(static_cast<int>(dst_width), 222 static_cast<int>(dst_height), 223 GetPixelWidth(), GetPixelHeight(), 224 GetTimeStamp()); 225 if (dest) { 226 StretchToFrame(dest, interpolate, vert_crop); 227 } 228 return dest; 229 } 230 231 bool VideoFrame::SetToBlack() { 232 return libyuv::I420Rect(GetYPlane(), GetYPitch(), 233 GetUPlane(), GetUPitch(), 234 GetVPlane(), GetVPitch(), 235 0, 0, 236 static_cast<int>(GetWidth()), 237 static_cast<int>(GetHeight()), 238 16, 128, 128) == 0; 239 } 240 241 static const size_t kMaxSampleSize = 1000000000u; 242 // Returns whether a sample is valid. 243 bool VideoFrame::Validate(uint32_t fourcc, 244 int w, 245 int h, 246 const uint8_t* sample, 247 size_t sample_size) { 248 if (h < 0) { 249 h = -h; 250 } 251 // 16384 is maximum resolution for VP8 codec. 252 if (w < 1 || w > 16384 || h < 1 || h > 16384) { 253 LOG(LS_ERROR) << "Invalid dimensions: " << w << "x" << h; 254 return false; 255 } 256 uint32_t format = CanonicalFourCC(fourcc); 257 int expected_bpp = 8; 258 switch (format) { 259 case FOURCC_I400: 260 case FOURCC_RGGB: 261 case FOURCC_BGGR: 262 case FOURCC_GRBG: 263 case FOURCC_GBRG: 264 expected_bpp = 8; 265 break; 266 case FOURCC_I420: 267 case FOURCC_I411: 268 case FOURCC_YU12: 269 case FOURCC_YV12: 270 case FOURCC_M420: 271 case FOURCC_NV21: 272 case FOURCC_NV12: 273 expected_bpp = 12; 274 break; 275 case FOURCC_I422: 276 case FOURCC_YV16: 277 case FOURCC_YUY2: 278 case FOURCC_UYVY: 279 case FOURCC_RGBP: 280 case FOURCC_RGBO: 281 case FOURCC_R444: 282 expected_bpp = 16; 283 break; 284 case FOURCC_I444: 285 case FOURCC_YV24: 286 case FOURCC_24BG: 287 case FOURCC_RAW: 288 expected_bpp = 24; 289 break; 290 291 case FOURCC_ABGR: 292 case FOURCC_BGRA: 293 case FOURCC_ARGB: 294 expected_bpp = 32; 295 break; 296 297 case FOURCC_MJPG: 298 case FOURCC_H264: 299 expected_bpp = 0; 300 break; 301 default: 302 expected_bpp = 8; // Expect format is at least 8 bits per pixel. 303 break; 304 } 305 size_t expected_size = (w * expected_bpp + 7) / 8 * h; 306 // For compressed formats, expect 4 bits per 16 x 16 macro. I420 would be 307 // 6 bits, but grey can be 4 bits. 308 if (expected_bpp == 0) { 309 expected_size = ((w + 15) / 16) * ((h + 15) / 16) * 4 / 8; 310 } 311 if (sample == NULL) { 312 LOG(LS_ERROR) << "NULL sample pointer." 313 << " format: " << GetFourccName(format) 314 << " bpp: " << expected_bpp 315 << " size: " << w << "x" << h 316 << " expected: " << expected_size 317 << " " << sample_size; 318 return false; 319 } 320 // TODO(fbarchard): Make function to dump information about frames. 321 uint8_t four_samples[4] = {0, 0, 0, 0}; 322 for (size_t i = 0; i < arraysize(four_samples) && i < sample_size; ++i) { 323 four_samples[i] = sample[i]; 324 } 325 if (sample_size < expected_size) { 326 LOG(LS_ERROR) << "Size field is too small." 327 << " format: " << GetFourccName(format) 328 << " bpp: " << expected_bpp 329 << " size: " << w << "x" << h 330 << " " << sample_size 331 << " expected: " << expected_size 332 << " sample[0..3]: " << static_cast<int>(four_samples[0]) 333 << ", " << static_cast<int>(four_samples[1]) 334 << ", " << static_cast<int>(four_samples[2]) 335 << ", " << static_cast<int>(four_samples[3]); 336 return false; 337 } 338 if (sample_size > kMaxSampleSize) { 339 LOG(LS_WARNING) << "Size field is invalid." 340 << " format: " << GetFourccName(format) 341 << " bpp: " << expected_bpp 342 << " size: " << w << "x" << h 343 << " " << sample_size 344 << " expected: " << 2 * expected_size 345 << " sample[0..3]: " << static_cast<int>(four_samples[0]) 346 << ", " << static_cast<int>(four_samples[1]) 347 << ", " << static_cast<int>(four_samples[2]) 348 << ", " << static_cast<int>(four_samples[3]); 349 return false; 350 } 351 // Show large size warning once every 100 frames. 352 // TODO(fbarchard): Make frame counter atomic for thread safety. 353 static int large_warn100 = 0; 354 size_t large_expected_size = expected_size * 2; 355 if (expected_bpp >= 8 && 356 (sample_size > large_expected_size || sample_size > kMaxSampleSize) && 357 large_warn100 % 100 == 0) { 358 ++large_warn100; 359 LOG(LS_WARNING) << "Size field is too large." 360 << " format: " << GetFourccName(format) 361 << " bpp: " << expected_bpp 362 << " size: " << w << "x" << h 363 << " bytes: " << sample_size 364 << " expected: " << large_expected_size 365 << " sample[0..3]: " << static_cast<int>(four_samples[0]) 366 << ", " << static_cast<int>(four_samples[1]) 367 << ", " << static_cast<int>(four_samples[2]) 368 << ", " << static_cast<int>(four_samples[3]); 369 } 370 371 // TODO(fbarchard): Add duplicate pixel check. 372 // TODO(fbarchard): Use frame counter atomic for thread safety. 373 static bool valid_once = true; 374 if (valid_once) { 375 valid_once = false; 376 LOG(LS_INFO) << "Validate frame passed." 377 << " format: " << GetFourccName(format) 378 << " bpp: " << expected_bpp 379 << " size: " << w << "x" << h 380 << " bytes: " << sample_size 381 << " expected: " << expected_size 382 << " sample[0..3]: " << static_cast<int>(four_samples[0]) 383 << ", " << static_cast<int>(four_samples[1]) 384 << ", " << static_cast<int>(four_samples[2]) 385 << ", " << static_cast<int>(four_samples[3]); 386 } 387 return true; 388 } 389 390 } // namespace cricket 391
