1 /* Copyright 2017 The Chromium OS Authors. All rights reserved. 2 * Use of this source code is governed by a BSD-style license that can be 3 * found in the LICENSE file. 4 */ 5 6 #include "arc/image_processor.h" 7 8 #include <cerrno> 9 #include <ctime> 10 #include <string> 11 12 #include <libyuv.h> 13 #include "arc/common.h" 14 #include "arc/exif_utils.h" 15 #include "arc/jpeg_compressor.h" 16 17 namespace arc { 18 19 using android::CameraMetadata; 20 21 /* 22 * Formats have different names in different header files. Here is the mapping 23 * table: 24 * 25 * android_pixel_format_t videodev2.h FOURCC in libyuv 26 * ----------------------------------------------------------------------------- 27 * HAL_PIXEL_FORMAT_YV12 = V4L2_PIX_FMT_YVU420 = FOURCC_YV12 28 * HAL_PIXEL_FORMAT_YCrCb_420_SP = V4L2_PIX_FMT_NV21 = FOURCC_NV21 29 * HAL_PIXEL_FORMAT_RGBA_8888 = V4L2_PIX_FMT_RGB32 = FOURCC_BGR4 30 * HAL_PIXEL_FORMAT_YCbCr_422_I = V4L2_PIX_FMT_YUYV = FOURCC_YUYV 31 * = FOURCC_YUY2 32 * V4L2_PIX_FMT_YUV420 = FOURCC_I420 33 * = FOURCC_YU12 34 * V4L2_PIX_FMT_MJPEG = FOURCC_MJPG 35 * 36 * Camera device generates FOURCC_YUYV and FOURCC_MJPG. 37 * Preview needs FOURCC_ARGB format. 38 * Software video encoder needs FOURCC_YU12. 39 * CTS requires FOURCC_YV12 and FOURCC_NV21 for applications. 40 * 41 * Android stride requirement: 42 * YV12 horizontal stride should be a multiple of 16 pixels. See 43 * android.graphics.ImageFormat.YV12. 44 * The stride of ARGB, YU12, and NV21 are always equal to the width. 45 * 46 * Conversion Path: 47 * MJPG/YUYV (from camera) -> YU12 -> ARGB (preview) 48 * -> NV21 (apps) 49 * -> YV12 (apps) 50 * -> YU12 (video encoder) 51 */ 52 53 // YV12 horizontal stride should be a multiple of 16 pixels for each plane. 54 // |dst_stride_uv| is the pixel stride of u or v plane. 55 static int YU12ToYV12(const void* yv12, void* yu12, int width, int height, 56 int dst_stride_y, int dst_stride_uv); 57 static int YU12ToNV21(const void* yv12, void* nv21, int width, int height); 58 static bool ConvertToJpeg(const CameraMetadata& metadata, 59 const FrameBuffer& in_frame, FrameBuffer* out_frame); 60 static bool SetExifTags(const CameraMetadata& metadata, ExifUtils* utils); 61 62 // How precise the float-to-rational conversion for EXIF tags would be. 63 static const int kRationalPrecision = 10000; 64 65 // Default JPEG quality settings. 66 static const int DEFAULT_JPEG_QUALITY = 80; 67 68 inline static size_t Align16(size_t value) { return (value + 15) & ~15; } 69 70 size_t ImageProcessor::GetConvertedSize(int fourcc, uint32_t width, 71 uint32_t height) { 72 if ((width % 2) || (height % 2)) { 73 LOGF(ERROR) << "Width or height is not even (" << width << " x " << height 74 << ")"; 75 return 0; 76 } 77 78 switch (fourcc) { 79 case V4L2_PIX_FMT_YVU420: // YV12 80 return Align16(width) * height + Align16(width / 2) * height; 81 case V4L2_PIX_FMT_YUV420: // YU12 82 // Fall-through. 83 case V4L2_PIX_FMT_NV21: // NV21 84 return width * height * 3 / 2; 85 case V4L2_PIX_FMT_BGR32: 86 case V4L2_PIX_FMT_RGB32: 87 return width * height * 4; 88 case V4L2_PIX_FMT_JPEG: 89 return 0; // For JPEG real size will be calculated after conversion. 90 default: 91 LOGF(ERROR) << "Pixel format " << FormatToString(fourcc) 92 << " is unsupported."; 93 return 0; 94 } 95 } 96 97 bool ImageProcessor::SupportsConversion(uint32_t from_fourcc, 98 uint32_t to_fourcc) { 99 switch (from_fourcc) { 100 case V4L2_PIX_FMT_YUYV: 101 return (to_fourcc == V4L2_PIX_FMT_YUV420); 102 case V4L2_PIX_FMT_YUV420: 103 return ( 104 to_fourcc == V4L2_PIX_FMT_YUV420 || 105 to_fourcc == V4L2_PIX_FMT_YVU420 || to_fourcc == V4L2_PIX_FMT_NV21 || 106 to_fourcc == V4L2_PIX_FMT_RGB32 || to_fourcc == V4L2_PIX_FMT_BGR32 || 107 to_fourcc == V4L2_PIX_FMT_JPEG); 108 case V4L2_PIX_FMT_MJPEG: 109 return (to_fourcc == V4L2_PIX_FMT_YUV420); 110 default: 111 return false; 112 } 113 } 114 115 int ImageProcessor::ConvertFormat(const CameraMetadata& metadata, 116 const FrameBuffer& in_frame, 117 FrameBuffer* out_frame) { 118 if ((in_frame.GetWidth() % 2) || (in_frame.GetHeight() % 2)) { 119 LOGF(ERROR) << "Width or height is not even (" << in_frame.GetWidth() 120 << " x " << in_frame.GetHeight() << ")"; 121 return -EINVAL; 122 } 123 124 size_t data_size = GetConvertedSize( 125 out_frame->GetFourcc(), in_frame.GetWidth(), in_frame.GetHeight()); 126 127 if (out_frame->SetDataSize(data_size)) { 128 LOGF(ERROR) << "Set data size failed"; 129 return -EINVAL; 130 } 131 132 if (in_frame.GetFourcc() == V4L2_PIX_FMT_YUYV) { 133 switch (out_frame->GetFourcc()) { 134 case V4L2_PIX_FMT_YUV420: // YU12 135 { 136 int res = libyuv::YUY2ToI420( 137 in_frame.GetData(), /* src_yuy2 */ 138 in_frame.GetWidth() * 2, /* src_stride_yuy2 */ 139 out_frame->GetData(), /* dst_y */ 140 out_frame->GetWidth(), /* dst_stride_y */ 141 out_frame->GetData() + 142 out_frame->GetWidth() * out_frame->GetHeight(), /* dst_u */ 143 out_frame->GetWidth() / 2, /* dst_stride_u */ 144 out_frame->GetData() + out_frame->GetWidth() * 145 out_frame->GetHeight() * 5 / 146 4, /* dst_v */ 147 out_frame->GetWidth() / 2, /* dst_stride_v */ 148 in_frame.GetWidth(), in_frame.GetHeight()); 149 LOGF_IF(ERROR, res) << "YUY2ToI420() for YU12 returns " << res; 150 return res ? -EINVAL : 0; 151 } 152 default: 153 LOGF(ERROR) << "Destination pixel format " 154 << FormatToString(out_frame->GetFourcc()) 155 << " is unsupported for YUYV source format."; 156 return -EINVAL; 157 } 158 } else if (in_frame.GetFourcc() == V4L2_PIX_FMT_YUV420) { 159 // V4L2_PIX_FMT_YVU420 is YV12. I420 is usually referred to YU12 160 // (V4L2_PIX_FMT_YUV420), and YV12 is similar to YU12 except that U/V 161 // planes are swapped. 162 switch (out_frame->GetFourcc()) { 163 case V4L2_PIX_FMT_YVU420: // YV12 164 { 165 int ystride = Align16(in_frame.GetWidth()); 166 int uvstride = Align16(in_frame.GetWidth() / 2); 167 int res = YU12ToYV12(in_frame.GetData(), out_frame->GetData(), 168 in_frame.GetWidth(), in_frame.GetHeight(), ystride, 169 uvstride); 170 LOGF_IF(ERROR, res) << "YU12ToYV12() returns " << res; 171 return res ? -EINVAL : 0; 172 } 173 case V4L2_PIX_FMT_YUV420: // YU12 174 { 175 memcpy(out_frame->GetData(), in_frame.GetData(), 176 in_frame.GetDataSize()); 177 return 0; 178 } 179 case V4L2_PIX_FMT_NV21: // NV21 180 { 181 // TODO(henryhsu): Use libyuv::I420ToNV21. 182 int res = YU12ToNV21(in_frame.GetData(), out_frame->GetData(), 183 in_frame.GetWidth(), in_frame.GetHeight()); 184 LOGF_IF(ERROR, res) << "YU12ToNV21() returns " << res; 185 return res ? -EINVAL : 0; 186 } 187 case V4L2_PIX_FMT_BGR32: { 188 int res = libyuv::I420ToABGR( 189 in_frame.GetData(), /* src_y */ 190 in_frame.GetWidth(), /* src_stride_y */ 191 in_frame.GetData() + 192 in_frame.GetWidth() * in_frame.GetHeight(), /* src_u */ 193 in_frame.GetWidth() / 2, /* src_stride_u */ 194 in_frame.GetData() + 195 in_frame.GetWidth() * in_frame.GetHeight() * 5 / 4, /* src_v */ 196 in_frame.GetWidth() / 2, /* src_stride_v */ 197 out_frame->GetData(), /* dst_abgr */ 198 out_frame->GetWidth() * 4, /* dst_stride_abgr */ 199 in_frame.GetWidth(), in_frame.GetHeight()); 200 LOGF_IF(ERROR, res) << "I420ToABGR() returns " << res; 201 return res ? -EINVAL : 0; 202 } 203 case V4L2_PIX_FMT_RGB32: { 204 int res = libyuv::I420ToARGB( 205 in_frame.GetData(), /* src_y */ 206 in_frame.GetWidth(), /* src_stride_y */ 207 in_frame.GetData() + 208 in_frame.GetWidth() * in_frame.GetHeight(), /* src_u */ 209 in_frame.GetWidth() / 2, /* src_stride_u */ 210 in_frame.GetData() + 211 in_frame.GetWidth() * in_frame.GetHeight() * 5 / 4, /* src_v */ 212 in_frame.GetWidth() / 2, /* src_stride_v */ 213 out_frame->GetData(), /* dst_argb */ 214 out_frame->GetWidth() * 4, /* dst_stride_argb */ 215 in_frame.GetWidth(), in_frame.GetHeight()); 216 LOGF_IF(ERROR, res) << "I420ToARGB() returns " << res; 217 return res ? -EINVAL : 0; 218 } 219 case V4L2_PIX_FMT_JPEG: { 220 bool res = ConvertToJpeg(metadata, in_frame, out_frame); 221 LOGF_IF(ERROR, !res) << "ConvertToJpeg() returns " << res; 222 return res ? -EINVAL : 0; 223 } 224 default: 225 LOGF(ERROR) << "Destination pixel format " 226 << FormatToString(out_frame->GetFourcc()) 227 << " is unsupported for YU12 source format."; 228 return -EINVAL; 229 } 230 } else if (in_frame.GetFourcc() == V4L2_PIX_FMT_MJPEG) { 231 switch (out_frame->GetFourcc()) { 232 case V4L2_PIX_FMT_YUV420: // YU12 233 { 234 int res = libyuv::MJPGToI420( 235 in_frame.GetData(), /* sample */ 236 in_frame.GetDataSize(), /* sample_size */ 237 out_frame->GetData(), /* dst_y */ 238 out_frame->GetWidth(), /* dst_stride_y */ 239 out_frame->GetData() + 240 out_frame->GetWidth() * out_frame->GetHeight(), /* dst_u */ 241 out_frame->GetWidth() / 2, /* dst_stride_u */ 242 out_frame->GetData() + out_frame->GetWidth() * 243 out_frame->GetHeight() * 5 / 244 4, /* dst_v */ 245 out_frame->GetWidth() / 2, /* dst_stride_v */ 246 in_frame.GetWidth(), in_frame.GetHeight(), out_frame->GetWidth(), 247 out_frame->GetHeight()); 248 LOGF_IF(ERROR, res) << "MJPEGToI420() returns " << res; 249 return res ? -EINVAL : 0; 250 } 251 default: 252 LOGF(ERROR) << "Destination pixel format " 253 << FormatToString(out_frame->GetFourcc()) 254 << " is unsupported for MJPEG source format."; 255 return -EINVAL; 256 } 257 } else { 258 LOGF(ERROR) << "Convert format doesn't support source format " 259 << FormatToString(in_frame.GetFourcc()); 260 return -EINVAL; 261 } 262 } 263 264 int ImageProcessor::Scale(const FrameBuffer& in_frame, FrameBuffer* out_frame) { 265 if (in_frame.GetFourcc() != V4L2_PIX_FMT_YUV420) { 266 LOGF(ERROR) << "Pixel format " << FormatToString(in_frame.GetFourcc()) 267 << " is unsupported."; 268 return -EINVAL; 269 } 270 271 size_t data_size = GetConvertedSize( 272 in_frame.GetFourcc(), out_frame->GetWidth(), out_frame->GetHeight()); 273 274 if (out_frame->SetDataSize(data_size)) { 275 LOGF(ERROR) << "Set data size failed"; 276 return -EINVAL; 277 } 278 out_frame->SetFourcc(in_frame.GetFourcc()); 279 280 VLOGF(1) << "Scale image from " << in_frame.GetWidth() << "x" 281 << in_frame.GetHeight() << " to " << out_frame->GetWidth() << "x" 282 << out_frame->GetHeight(); 283 284 int ret = libyuv::I420Scale( 285 in_frame.GetData(), in_frame.GetWidth(), 286 in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight(), 287 in_frame.GetWidth() / 2, 288 in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight() * 5 / 4, 289 in_frame.GetWidth() / 2, in_frame.GetWidth(), in_frame.GetHeight(), 290 out_frame->GetData(), out_frame->GetWidth(), 291 out_frame->GetData() + out_frame->GetWidth() * out_frame->GetHeight(), 292 out_frame->GetWidth() / 2, 293 out_frame->GetData() + 294 out_frame->GetWidth() * out_frame->GetHeight() * 5 / 4, 295 out_frame->GetWidth() / 2, out_frame->GetWidth(), out_frame->GetHeight(), 296 libyuv::FilterMode::kFilterNone); 297 LOGF_IF(ERROR, ret) << "I420Scale failed: " << ret; 298 return ret; 299 } 300 301 static int YU12ToYV12(const void* yu12, void* yv12, int width, int height, 302 int dst_stride_y, int dst_stride_uv) { 303 if ((width % 2) || (height % 2)) { 304 LOGF(ERROR) << "Width or height is not even (" << width << " x " << height 305 << ")"; 306 return -EINVAL; 307 } 308 if (dst_stride_y < width || dst_stride_uv < width / 2) { 309 LOGF(ERROR) << "Y plane stride (" << dst_stride_y 310 << ") or U/V plane stride (" << dst_stride_uv 311 << ") is invalid for width " << width; 312 return -EINVAL; 313 } 314 315 const uint8_t* src = reinterpret_cast<const uint8_t*>(yu12); 316 uint8_t* dst = reinterpret_cast<uint8_t*>(yv12); 317 const uint8_t* u_src = src + width * height; 318 uint8_t* u_dst = dst + dst_stride_y * height + dst_stride_uv * height / 2; 319 const uint8_t* v_src = src + width * height * 5 / 4; 320 uint8_t* v_dst = dst + dst_stride_y * height; 321 322 return libyuv::I420Copy(src, width, u_src, width / 2, v_src, width / 2, dst, 323 dst_stride_y, u_dst, dst_stride_uv, v_dst, 324 dst_stride_uv, width, height); 325 } 326 327 static int YU12ToNV21(const void* yu12, void* nv21, int width, int height) { 328 if ((width % 2) || (height % 2)) { 329 LOGF(ERROR) << "Width or height is not even (" << width << " x " << height 330 << ")"; 331 return -EINVAL; 332 } 333 334 const uint8_t* src = reinterpret_cast<const uint8_t*>(yu12); 335 uint8_t* dst = reinterpret_cast<uint8_t*>(nv21); 336 const uint8_t* u_src = src + width * height; 337 const uint8_t* v_src = src + width * height * 5 / 4; 338 uint8_t* vu_dst = dst + width * height; 339 340 memcpy(dst, src, width * height); 341 342 for (int i = 0; i < height / 2; i++) { 343 for (int j = 0; j < width / 2; j++) { 344 *vu_dst++ = *v_src++; 345 *vu_dst++ = *u_src++; 346 } 347 } 348 return 0; 349 } 350 351 static bool ConvertToJpeg(const CameraMetadata& metadata, 352 const FrameBuffer& in_frame, FrameBuffer* out_frame) { 353 ExifUtils utils; 354 int jpeg_quality, thumbnail_jpeg_quality; 355 camera_metadata_ro_entry entry; 356 357 if (metadata.exists(ANDROID_JPEG_QUALITY)) { 358 entry = metadata.find(ANDROID_JPEG_QUALITY); 359 jpeg_quality = entry.data.u8[0]; 360 } else { 361 LOGF(ERROR) << "Could not find jpeg quality in metadata, defaulting to " 362 << DEFAULT_JPEG_QUALITY; 363 jpeg_quality = DEFAULT_JPEG_QUALITY; 364 } 365 if (metadata.exists(ANDROID_JPEG_THUMBNAIL_QUALITY)) { 366 entry = metadata.find(ANDROID_JPEG_THUMBNAIL_QUALITY); 367 thumbnail_jpeg_quality = entry.data.u8[0]; 368 } else { 369 thumbnail_jpeg_quality = jpeg_quality; 370 } 371 372 if (!utils.Initialize(in_frame.GetData(), in_frame.GetWidth(), 373 in_frame.GetHeight(), thumbnail_jpeg_quality)) { 374 LOGF(ERROR) << "ExifUtils initialization failed."; 375 return false; 376 } 377 if (!SetExifTags(metadata, &utils)) { 378 LOGF(ERROR) << "Setting Exif tags failed."; 379 return false; 380 } 381 if (!utils.GenerateApp1()) { 382 LOGF(ERROR) << "Generating APP1 segment failed."; 383 return false; 384 } 385 JpegCompressor compressor; 386 if (!compressor.CompressImage(in_frame.GetData(), in_frame.GetWidth(), 387 in_frame.GetHeight(), jpeg_quality, 388 utils.GetApp1Buffer(), utils.GetApp1Length())) { 389 LOGF(ERROR) << "JPEG image compression failed"; 390 return false; 391 } 392 size_t buffer_length = compressor.GetCompressedImageSize(); 393 if (out_frame->SetDataSize(buffer_length)) { 394 return false; 395 } 396 memcpy(out_frame->GetData(), compressor.GetCompressedImagePtr(), 397 buffer_length); 398 return true; 399 } 400 401 static bool SetExifTags(const CameraMetadata& metadata, ExifUtils* utils) { 402 time_t raw_time = 0; 403 struct tm time_info; 404 bool time_available = time(&raw_time) != -1; 405 localtime_r(&raw_time, &time_info); 406 if (!utils->SetDateTime(time_info)) { 407 LOGF(ERROR) << "Setting data time failed."; 408 return false; 409 } 410 411 float focal_length; 412 camera_metadata_ro_entry entry = metadata.find(ANDROID_LENS_FOCAL_LENGTH); 413 if (entry.count) { 414 focal_length = entry.data.f[0]; 415 } else { 416 LOGF(ERROR) << "Cannot find focal length in metadata."; 417 return false; 418 } 419 if (!utils->SetFocalLength( 420 static_cast<uint32_t>(focal_length * kRationalPrecision), 421 kRationalPrecision)) { 422 LOGF(ERROR) << "Setting focal length failed."; 423 return false; 424 } 425 426 if (metadata.exists(ANDROID_JPEG_GPS_COORDINATES)) { 427 entry = metadata.find(ANDROID_JPEG_GPS_COORDINATES); 428 if (entry.count < 3) { 429 LOGF(ERROR) << "Gps coordinates in metadata is not complete."; 430 return false; 431 } 432 if (!utils->SetGpsLatitude(entry.data.d[0])) { 433 LOGF(ERROR) << "Setting gps latitude failed."; 434 return false; 435 } 436 if (!utils->SetGpsLongitude(entry.data.d[1])) { 437 LOGF(ERROR) << "Setting gps longitude failed."; 438 return false; 439 } 440 if (!utils->SetGpsAltitude(entry.data.d[2])) { 441 LOGF(ERROR) << "Setting gps altitude failed."; 442 return false; 443 } 444 } 445 446 if (metadata.exists(ANDROID_JPEG_GPS_PROCESSING_METHOD)) { 447 entry = metadata.find(ANDROID_JPEG_GPS_PROCESSING_METHOD); 448 std::string method_str(reinterpret_cast<const char*>(entry.data.u8)); 449 if (!utils->SetGpsProcessingMethod(method_str)) { 450 LOGF(ERROR) << "Setting gps processing method failed."; 451 return false; 452 } 453 } 454 455 if (time_available && metadata.exists(ANDROID_JPEG_GPS_TIMESTAMP)) { 456 entry = metadata.find(ANDROID_JPEG_GPS_TIMESTAMP); 457 time_t timestamp = static_cast<time_t>(entry.data.i64[0]); 458 if (gmtime_r(×tamp, &time_info)) { 459 if (!utils->SetGpsTimestamp(time_info)) { 460 LOGF(ERROR) << "Setting gps timestamp failed."; 461 return false; 462 } 463 } else { 464 LOGF(ERROR) << "Time tranformation failed."; 465 return false; 466 } 467 } 468 469 if (metadata.exists(ANDROID_JPEG_ORIENTATION)) { 470 entry = metadata.find(ANDROID_JPEG_ORIENTATION); 471 if (!utils->SetOrientation(entry.data.i32[0])) { 472 LOGF(ERROR) << "Setting orientation failed."; 473 return false; 474 } 475 } 476 477 if (metadata.exists(ANDROID_JPEG_THUMBNAIL_SIZE)) { 478 entry = metadata.find(ANDROID_JPEG_THUMBNAIL_SIZE); 479 if (entry.count < 2) { 480 LOGF(ERROR) << "Thumbnail size in metadata is not complete."; 481 return false; 482 } 483 int thumbnail_width = entry.data.i32[0]; 484 int thumbnail_height = entry.data.i32[1]; 485 if (thumbnail_width > 0 && thumbnail_height > 0) { 486 if (!utils->SetThumbnailSize(static_cast<uint16_t>(thumbnail_width), 487 static_cast<uint16_t>(thumbnail_height))) { 488 LOGF(ERROR) << "Setting thumbnail size failed."; 489 return false; 490 } 491 } 492 } 493 return true; 494 } 495 496 } // namespace arc 497
