1 /* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "NuPlayerCCDecoder" 19 #include <utils/Log.h> 20 #include <inttypes.h> 21 22 #include "NuPlayerCCDecoder.h" 23 24 #include <media/stagefright/foundation/ABitReader.h> 25 #include <media/stagefright/foundation/ABuffer.h> 26 #include <media/stagefright/foundation/ADebug.h> 27 #include <media/stagefright/foundation/AMessage.h> 28 #include <media/stagefright/foundation/avc_utils.h> 29 #include <media/stagefright/MediaDefs.h> 30 31 namespace android { 32 33 // In CEA-708B, the maximum bandwidth of CC is set to 9600bps. 34 static const size_t kMaxBandwithSizeBytes = 9600 / 8; 35 36 struct CCData { 37 CCData(uint8_t type, uint8_t data1, uint8_t data2) 38 : mType(type), mData1(data1), mData2(data2) { 39 } 40 bool getChannel(size_t *channel) const { 41 if (mData1 >= 0x10 && mData1 <= 0x1f) { 42 *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0); 43 return true; 44 } 45 return false; 46 } 47 48 uint8_t mType; 49 uint8_t mData1; 50 uint8_t mData2; 51 }; 52 53 static bool isNullPad(CCData *cc) { 54 return cc->mData1 < 0x10 && cc->mData2 < 0x10; 55 } 56 57 static void dumpBytePair(const sp<ABuffer> &ccBuf) __attribute__ ((unused)); 58 static void dumpBytePair(const sp<ABuffer> &ccBuf) { 59 size_t offset = 0; 60 AString out; 61 62 while (offset < ccBuf->size()) { 63 char tmp[128]; 64 65 CCData *cc = (CCData *) (ccBuf->data() + offset); 66 67 if (isNullPad(cc)) { 68 // 1 null pad or XDS metadata, ignore 69 offset += sizeof(CCData); 70 continue; 71 } 72 73 if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) { 74 // 2 basic chars 75 snprintf(tmp, sizeof(tmp), "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2); 76 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) 77 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) { 78 // 1 special char 79 snprintf(tmp, sizeof(tmp), "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2); 80 } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A) 81 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ 82 // 1 Spanish/French char 83 snprintf(tmp, sizeof(tmp), "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2); 84 } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B) 85 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ 86 // 1 Portuguese/German/Danish char 87 snprintf(tmp, sizeof(tmp), "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2); 88 } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) 89 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){ 90 // Mid-Row Codes (Table 69) 91 snprintf(tmp, sizeof(tmp), "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2); 92 } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c) 93 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f) 94 || 95 ((cc->mData1 == 0x17 || cc->mData1 == 0x1f) 96 && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){ 97 // Misc Control Codes (Table 70) 98 snprintf(tmp, sizeof(tmp), "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2); 99 } else if ((cc->mData1 & 0x70) == 0x10 100 && (cc->mData2 & 0x40) == 0x40 101 && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) { 102 // Preamble Address Codes (Table 71) 103 snprintf(tmp, sizeof(tmp), "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2); 104 } else { 105 snprintf(tmp, sizeof(tmp), "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2); 106 } 107 108 if (out.size() > 0) { 109 out.append(", "); 110 } 111 112 out.append(tmp); 113 114 offset += sizeof(CCData); 115 } 116 117 ALOGI("%s", out.c_str()); 118 } 119 120 NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify) 121 : mNotify(notify), 122 mSelectedTrack(-1), 123 mDTVCCPacket(new ABuffer(kMaxBandwithSizeBytes)) { 124 mDTVCCPacket->setRange(0, 0); 125 126 // In CEA-608, streams from packets which have the value 0 of cc_type contain CC1 and CC2, and 127 // streams from packets which have the value 1 of cc_type contain CC3 and CC4. 128 // The following array indicates the current transmitting channels for each value of cc_type. 129 mLine21Channels[0] = 0; // CC1 130 mLine21Channels[1] = 2; // CC3 131 } 132 133 size_t NuPlayer::CCDecoder::getTrackCount() const { 134 return mTracks.size(); 135 } 136 137 sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const { 138 if (!isTrackValid(index)) { 139 return NULL; 140 } 141 142 sp<AMessage> format = new AMessage(); 143 144 CCTrack track = mTracks[index]; 145 146 format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); 147 format->setString("language", "und"); 148 149 switch (track.mTrackType) { 150 case kTrackTypeCEA608: 151 format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); 152 break; 153 case kTrackTypeCEA708: 154 format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_708); 155 break; 156 default: 157 ALOGE("Unknown track type: %d", track.mTrackType); 158 format->setInt32("type", MEDIA_TRACK_TYPE_UNKNOWN); 159 format->setString("mime", "application/octet-stream"); 160 return format; 161 } 162 163 // For CEA-608 CC1, field 0 channel 0 164 bool isDefaultAuto = track.mTrackType == kTrackTypeCEA608 165 && track.mTrackChannel == 0; 166 // For CEA-708, Primary Caption Service. 167 bool isDefaultOnly = track.mTrackType == kTrackTypeCEA708 168 && track.mTrackChannel == 1; 169 format->setInt32("auto", isDefaultAuto); 170 format->setInt32("default", isDefaultAuto || isDefaultOnly); 171 format->setInt32("forced", 0); 172 173 return format; 174 } 175 176 status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { 177 if (!isTrackValid(index)) { 178 return BAD_VALUE; 179 } 180 181 if (select) { 182 if (mSelectedTrack == (ssize_t)index) { 183 ALOGE("track %zu already selected", index); 184 return BAD_VALUE; 185 } 186 ALOGV("selected track %zu", index); 187 mSelectedTrack = index; 188 } else { 189 if (mSelectedTrack != (ssize_t)index) { 190 ALOGE("track %zu is not selected", index); 191 return BAD_VALUE; 192 } 193 ALOGV("unselected track %zu", index); 194 mSelectedTrack = -1; 195 } 196 197 // Clear the previous track payloads 198 mCCMap.clear(); 199 200 return OK; 201 } 202 203 bool NuPlayer::CCDecoder::isSelected() const { 204 return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)getTrackCount(); 205 } 206 207 bool NuPlayer::CCDecoder::isTrackValid(size_t index) const { 208 return index < getTrackCount(); 209 } 210 211 // returns true if a new CC track is found 212 bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { 213 sp<ABuffer> sei; 214 if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) { 215 return false; 216 } 217 218 int64_t timeUs; 219 CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); 220 221 bool trackAdded = false; 222 223 const NALPosition *nal = (NALPosition *)sei->data(); 224 225 for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) { 226 trackAdded |= parseSEINalUnit( 227 timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize); 228 } 229 230 return trackAdded; 231 } 232 233 // returns true if a new CC track is found 234 bool NuPlayer::CCDecoder::parseSEINalUnit(int64_t timeUs, const uint8_t *data, size_t size) { 235 unsigned nalType = data[0] & 0x1f; 236 237 // the buffer should only have SEI in it 238 if (nalType != 6) { 239 return false; 240 } 241 242 bool trackAdded = false; 243 NALBitReader br(data + 1, size - 1); 244 245 // sei_message() 246 while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message() 247 uint32_t payload_type = 0; 248 size_t payload_size = 0; 249 uint8_t last_byte; 250 251 do { 252 last_byte = br.getBits(8); 253 payload_type += last_byte; 254 } while (last_byte == 0xFF); 255 256 do { 257 last_byte = br.getBits(8); 258 payload_size += last_byte; 259 } while (last_byte == 0xFF); 260 261 if (payload_size > SIZE_MAX / 8 262 || !br.atLeastNumBitsLeft(payload_size * 8)) { 263 ALOGV("Malformed SEI payload"); 264 break; 265 } 266 267 // sei_payload() 268 if (payload_type == 4) { 269 bool isCC = false; 270 if (payload_size > 1 + 2 + 4 + 1) { 271 // user_data_registered_itu_t_t35() 272 273 // ATSC A/72: 6.4.2 274 uint8_t itu_t_t35_country_code = br.getBits(8); 275 uint16_t itu_t_t35_provider_code = br.getBits(16); 276 uint32_t user_identifier = br.getBits(32); 277 uint8_t user_data_type_code = br.getBits(8); 278 279 payload_size -= 1 + 2 + 4 + 1; 280 281 isCC = itu_t_t35_country_code == 0xB5 282 && itu_t_t35_provider_code == 0x0031 283 && user_identifier == 'GA94' 284 && user_data_type_code == 0x3; 285 } 286 287 if (isCC && payload_size > 2) { 288 trackAdded |= parseMPEGCCData(timeUs, br.data(), br.numBitsLeft() / 8); 289 } else { 290 ALOGV("Malformed SEI payload type 4"); 291 } 292 } else { 293 ALOGV("Unsupported SEI payload type %d", payload_type); 294 } 295 296 // skipping remaining bits of this payload 297 br.skipBits(payload_size * 8); 298 } 299 300 return trackAdded; 301 } 302 303 // returns true if a new CC track is found 304 bool NuPlayer::CCDecoder::extractFromMPEGUserData(const sp<ABuffer> &accessUnit) { 305 sp<ABuffer> mpegUserData; 306 if (!accessUnit->meta()->findBuffer("mpeg-user-data", &mpegUserData) 307 || mpegUserData == NULL) { 308 return false; 309 } 310 311 int64_t timeUs; 312 CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); 313 314 bool trackAdded = false; 315 316 const size_t *userData = (size_t *)mpegUserData->data(); 317 318 for (size_t i = 0; i < mpegUserData->size() / sizeof(size_t); ++i) { 319 trackAdded |= parseMPEGUserDataUnit( 320 timeUs, accessUnit->data() + userData[i], accessUnit->size() - userData[i]); 321 } 322 323 return trackAdded; 324 } 325 326 // returns true if a new CC track is found 327 bool NuPlayer::CCDecoder::parseMPEGUserDataUnit(int64_t timeUs, const uint8_t *data, size_t size) { 328 ABitReader br(data + 4, 5); 329 330 uint32_t user_identifier = br.getBits(32); 331 uint8_t user_data_type = br.getBits(8); 332 333 if (user_identifier == 'GA94' && user_data_type == 0x3) { 334 return parseMPEGCCData(timeUs, data + 9, size - 9); 335 } 336 337 return false; 338 } 339 340 // returns true if a new CC track is found 341 bool NuPlayer::CCDecoder::parseMPEGCCData(int64_t timeUs, const uint8_t *data, size_t size) { 342 bool trackAdded = false; 343 344 // MPEG_cc_data() 345 // ATSC A/53 Part 4: 6.2.3.1 346 ABitReader br(data, size); 347 348 if (br.numBitsLeft() <= 16) { 349 return false; 350 } 351 352 br.skipBits(1); 353 bool process_cc_data_flag = br.getBits(1); 354 br.skipBits(1); 355 size_t cc_count = br.getBits(5); 356 br.skipBits(8); 357 358 if (!process_cc_data_flag || 3 * 8 * cc_count >= br.numBitsLeft()) { 359 return false; 360 } 361 362 sp<ABuffer> line21CCBuf = NULL; 363 364 for (size_t i = 0; i < cc_count; ++i) { 365 br.skipBits(5); 366 bool cc_valid = br.getBits(1); 367 uint8_t cc_type = br.getBits(2); 368 369 if (cc_valid) { 370 if (cc_type == 3) { 371 if (mDTVCCPacket->size() > 0) { 372 trackAdded |= parseDTVCCPacket( 373 timeUs, mDTVCCPacket->data(), mDTVCCPacket->size()); 374 mDTVCCPacket->setRange(0, 0); 375 } 376 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2); 377 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2); 378 br.skipBits(16); 379 } else if (mDTVCCPacket->size() > 0 && cc_type == 2) { 380 memcpy(mDTVCCPacket->data() + mDTVCCPacket->size(), br.data(), 2); 381 mDTVCCPacket->setRange(0, mDTVCCPacket->size() + 2); 382 br.skipBits(16); 383 } else if (cc_type == 0 || cc_type == 1) { 384 uint8_t cc_data_1 = br.getBits(8) & 0x7f; 385 uint8_t cc_data_2 = br.getBits(8) & 0x7f; 386 387 CCData cc(cc_type, cc_data_1, cc_data_2); 388 389 if (isNullPad(&cc)) { 390 continue; 391 } 392 393 size_t channel; 394 if (cc.getChannel(&channel)) { 395 mLine21Channels[cc_type] = channel; 396 397 // create a new track if it does not exist. 398 getTrackIndex(kTrackTypeCEA608, channel, &trackAdded); 399 } 400 401 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608 402 && mTracks[mSelectedTrack].mTrackChannel == mLine21Channels[cc_type]) { 403 if (line21CCBuf == NULL) { 404 line21CCBuf = new ABuffer((cc_count - i) * sizeof(CCData)); 405 line21CCBuf->setRange(0, 0); 406 } 407 memcpy(line21CCBuf->data() + line21CCBuf->size(), &cc, sizeof(cc)); 408 line21CCBuf->setRange(0, line21CCBuf->size() + sizeof(CCData)); 409 } 410 } else { 411 br.skipBits(16); 412 } 413 } else { 414 if ((cc_type == 3 || cc_type == 2) && mDTVCCPacket->size() > 0) { 415 trackAdded |= parseDTVCCPacket(timeUs, mDTVCCPacket->data(), mDTVCCPacket->size()); 416 mDTVCCPacket->setRange(0, 0); 417 } 418 br.skipBits(16); 419 } 420 } 421 422 if (isSelected() && mTracks[mSelectedTrack].mTrackType == kTrackTypeCEA608 423 && line21CCBuf != NULL && line21CCBuf->size() > 0) { 424 mCCMap.add(timeUs, line21CCBuf); 425 } 426 427 return trackAdded; 428 } 429 430 // returns true if a new CC track is found 431 bool NuPlayer::CCDecoder::parseDTVCCPacket(int64_t timeUs, const uint8_t *data, size_t size) { 432 // CEA-708B 5 DTVCC Packet Layer. 433 ABitReader br(data, size); 434 br.skipBits(2); 435 436 size_t packet_size = br.getBits(6); 437 if (packet_size == 0) packet_size = 64; 438 packet_size *= 2; 439 440 if (size != packet_size) { 441 return false; 442 } 443 444 bool trackAdded = false; 445 446 while (br.numBitsLeft() >= 16) { 447 // CEA-708B Figure 5 and 6. 448 uint8_t service_number = br.getBits(3); 449 size_t block_size = br.getBits(5); 450 451 if (service_number == 64) { 452 br.skipBits(2); 453 service_number = br.getBits(6); 454 455 if (service_number < 64) { 456 return trackAdded; 457 } 458 } 459 460 if (br.numBitsLeft() < block_size * 8) { 461 return trackAdded; 462 } 463 464 if (block_size > 0) { 465 size_t trackIndex = getTrackIndex(kTrackTypeCEA708, service_number, &trackAdded); 466 if (mSelectedTrack == (ssize_t)trackIndex) { 467 sp<ABuffer> ccPacket = new ABuffer(block_size); 468 memcpy(ccPacket->data(), br.data(), block_size); 469 mCCMap.add(timeUs, ccPacket); 470 } 471 } 472 br.skipBits(block_size * 8); 473 } 474 475 return trackAdded; 476 } 477 478 // return the track index for a given type and channel. 479 // if the track does not exist, creates a new one. 480 size_t NuPlayer::CCDecoder::getTrackIndex( 481 int32_t trackType, size_t channel, bool *trackAdded) { 482 CCTrack track(trackType, channel); 483 ssize_t index = mTrackIndices.indexOfKey(track); 484 485 if (index < 0) { 486 // A new track is added. 487 index = mTracks.size(); 488 mTrackIndices.add(track, index); 489 mTracks.add(track); 490 *trackAdded = true; 491 return index; 492 } 493 494 return mTrackIndices.valueAt(index); 495 } 496 497 void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) { 498 if (extractFromMPEGUserData(accessUnit) || extractFromSEI(accessUnit)) { 499 sp<AMessage> msg = mNotify->dup(); 500 msg->setInt32("what", kWhatTrackAdded); 501 msg->post(); 502 } 503 // TODO: extract CC from other sources 504 } 505 506 void NuPlayer::CCDecoder::display(int64_t timeUs) { 507 if (!isSelected()) { 508 return; 509 } 510 511 ssize_t index = mCCMap.indexOfKey(timeUs); 512 if (index < 0) { 513 ALOGV("cc for timestamp %" PRId64 " not found", timeUs); 514 return; 515 } 516 517 sp<ABuffer> ccBuf; 518 519 if (index == 0) { 520 ccBuf = mCCMap.valueAt(index); 521 } else { 522 size_t size = 0; 523 524 for (ssize_t i = 0; i <= index; ++i) { 525 size += mCCMap.valueAt(i)->size(); 526 } 527 528 ccBuf = new ABuffer(size); 529 ccBuf->setRange(0, 0); 530 531 for (ssize_t i = 0; i <= index; ++i) { 532 sp<ABuffer> buf = mCCMap.valueAt(i); 533 memcpy(ccBuf->data() + ccBuf->size(), buf->data(), buf->size()); 534 ccBuf->setRange(0, ccBuf->size() + buf->size()); 535 } 536 } 537 538 if (ccBuf->size() > 0) { 539 #if 0 540 dumpBytePair(ccBuf); 541 #endif 542 543 ccBuf->meta()->setInt32("track-index", mSelectedTrack); 544 ccBuf->meta()->setInt64("timeUs", timeUs); 545 ccBuf->meta()->setInt64("durationUs", 0ll); 546 547 sp<AMessage> msg = mNotify->dup(); 548 msg->setInt32("what", kWhatClosedCaptionData); 549 msg->setBuffer("buffer", ccBuf); 550 msg->post(); 551 } 552 553 // remove all entries before timeUs 554 mCCMap.removeItemsAt(0, index + 1); 555 } 556 557 void NuPlayer::CCDecoder::flush() { 558 mCCMap.clear(); 559 mDTVCCPacket->setRange(0, 0); 560 } 561 562 int32_t NuPlayer::CCDecoder::CCTrack::compare(const NuPlayer::CCDecoder::CCTrack& rhs) const { 563 int32_t cmp = mTrackType - rhs.mTrackType; 564 if (cmp != 0) return cmp; 565 return mTrackChannel - rhs.mTrackChannel; 566 } 567 568 bool NuPlayer::CCDecoder::CCTrack::operator<(const NuPlayer::CCDecoder::CCTrack& rhs) const { 569 return compare(rhs) < 0; 570 } 571 572 bool NuPlayer::CCDecoder::CCTrack::operator==(const NuPlayer::CCDecoder::CCTrack& rhs) const { 573 return compare(rhs) == 0; 574 } 575 576 bool NuPlayer::CCDecoder::CCTrack::operator!=(const NuPlayer::CCDecoder::CCTrack& rhs) const { 577 return compare(rhs) != 0; 578 } 579 580 } // namespace android 581 582