1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.graphics.Canvas; 20 import android.media.MediaPlayer.TrackInfo; 21 import android.os.Handler; 22 import android.util.Log; 23 import android.util.LongSparseArray; 24 import android.util.Pair; 25 26 import java.util.Iterator; 27 import java.util.NoSuchElementException; 28 import java.util.SortedMap; 29 import java.util.TreeMap; 30 import java.util.Vector; 31 32 /** 33 * A subtitle track abstract base class that is responsible for parsing and displaying 34 * an instance of a particular type of subtitle. 35 * 36 * @hide 37 */ 38 public abstract class SubtitleTrack implements MediaTimeProvider.OnMediaTimeListener { 39 private static final String TAG = "SubtitleTrack"; 40 private long mLastUpdateTimeMs; 41 private long mLastTimeMs; 42 43 private Runnable mRunnable; 44 45 /** @hide TODO private */ 46 final protected LongSparseArray<Run> mRunsByEndTime = new LongSparseArray<Run>(); 47 /** @hide TODO private */ 48 final protected LongSparseArray<Run> mRunsByID = new LongSparseArray<Run>(); 49 50 /** @hide TODO private */ 51 protected CueList mCues; 52 /** @hide TODO private */ 53 final protected Vector<Cue> mActiveCues = new Vector<Cue>(); 54 /** @hide */ 55 protected boolean mVisible; 56 57 /** @hide */ 58 public boolean DEBUG = false; 59 60 /** @hide */ 61 protected Handler mHandler = new Handler(); 62 63 private MediaFormat mFormat; 64 65 public SubtitleTrack(MediaFormat format) { 66 mFormat = format; 67 mCues = new CueList(); 68 clearActiveCues(); 69 mLastTimeMs = -1; 70 } 71 72 /** @hide */ 73 public final MediaFormat getFormat() { 74 return mFormat; 75 } 76 77 private long mNextScheduledTimeMs = -1; 78 79 protected void onData(SubtitleData data) { 80 long runID = data.getStartTimeUs() + 1; 81 onData(data.getData(), true /* eos */, runID); 82 setRunDiscardTimeMs( 83 runID, 84 (data.getStartTimeUs() + data.getDurationUs()) / 1000); 85 } 86 87 /** 88 * Called when there is input data for the subtitle track. The 89 * complete subtitle for a track can include multiple whole units 90 * (runs). Each of these units can have multiple sections. The 91 * contents of a run are submitted in sequential order, with eos 92 * indicating the last section of the run. Calls from different 93 * runs must not be intermixed. 94 * 95 * @param data subtitle data byte buffer 96 * @param eos true if this is the last section of the run. 97 * @param runID mostly-unique ID for this run of data. Subtitle cues 98 * with runID of 0 are discarded immediately after 99 * display. Cues with runID of ~0 are discarded 100 * only at the deletion of the track object. Cues 101 * with other runID-s are discarded at the end of the 102 * run, which defaults to the latest timestamp of 103 * any of its cues (with this runID). 104 */ 105 public abstract void onData(byte[] data, boolean eos, long runID); 106 107 /** 108 * Called when adding the subtitle rendering widget to the view hierarchy, 109 * as well as when showing or hiding the subtitle track, or when the video 110 * surface position has changed. 111 * 112 * @return the widget that renders this subtitle track. For most renderers 113 * there should be a single shared instance that is used for all 114 * tracks supported by that renderer, as at most one subtitle track 115 * is visible at one time. 116 */ 117 public abstract RenderingWidget getRenderingWidget(); 118 119 /** 120 * Called when the active cues have changed, and the contents of the subtitle 121 * view should be updated. 122 * 123 * @hide 124 */ 125 public abstract void updateView(Vector<Cue> activeCues); 126 127 /** @hide */ 128 protected synchronized void updateActiveCues(boolean rebuild, long timeMs) { 129 // out-of-order times mean seeking or new active cues being added 130 // (during their own timespan) 131 if (rebuild || mLastUpdateTimeMs > timeMs) { 132 clearActiveCues(); 133 } 134 135 for(Iterator<Pair<Long, Cue> > it = 136 mCues.entriesBetween(mLastUpdateTimeMs, timeMs).iterator(); it.hasNext(); ) { 137 Pair<Long, Cue> event = it.next(); 138 Cue cue = event.second; 139 140 if (cue.mEndTimeMs == event.first) { 141 // remove past cues 142 if (DEBUG) Log.v(TAG, "Removing " + cue); 143 mActiveCues.remove(cue); 144 if (cue.mRunID == 0) { 145 it.remove(); 146 } 147 } else if (cue.mStartTimeMs == event.first) { 148 // add new cues 149 // TRICKY: this will happen in start order 150 if (DEBUG) Log.v(TAG, "Adding " + cue); 151 if (cue.mInnerTimesMs != null) { 152 cue.onTime(timeMs); 153 } 154 mActiveCues.add(cue); 155 } else if (cue.mInnerTimesMs != null) { 156 // cue is modified 157 cue.onTime(timeMs); 158 } 159 } 160 161 /* complete any runs */ 162 while (mRunsByEndTime.size() > 0 && 163 mRunsByEndTime.keyAt(0) <= timeMs) { 164 removeRunsByEndTimeIndex(0); // removes element 165 } 166 mLastUpdateTimeMs = timeMs; 167 } 168 169 private void removeRunsByEndTimeIndex(int ix) { 170 Run run = mRunsByEndTime.valueAt(ix); 171 while (run != null) { 172 Cue cue = run.mFirstCue; 173 while (cue != null) { 174 mCues.remove(cue); 175 Cue nextCue = cue.mNextInRun; 176 cue.mNextInRun = null; 177 cue = nextCue; 178 } 179 mRunsByID.remove(run.mRunID); 180 Run nextRun = run.mNextRunAtEndTimeMs; 181 run.mPrevRunAtEndTimeMs = null; 182 run.mNextRunAtEndTimeMs = null; 183 run = nextRun; 184 } 185 mRunsByEndTime.removeAt(ix); 186 } 187 188 @Override 189 protected void finalize() throws Throwable { 190 /* remove all cues (untangle all cross-links) */ 191 int size = mRunsByEndTime.size(); 192 for(int ix = size - 1; ix >= 0; ix--) { 193 removeRunsByEndTimeIndex(ix); 194 } 195 196 super.finalize(); 197 } 198 199 private synchronized void takeTime(long timeMs) { 200 mLastTimeMs = timeMs; 201 } 202 203 /** @hide */ 204 protected synchronized void clearActiveCues() { 205 if (DEBUG) Log.v(TAG, "Clearing " + mActiveCues.size() + " active cues"); 206 mActiveCues.clear(); 207 mLastUpdateTimeMs = -1; 208 } 209 210 /** @hide */ 211 protected void scheduleTimedEvents() { 212 /* get times for the next event */ 213 if (mTimeProvider != null) { 214 mNextScheduledTimeMs = mCues.nextTimeAfter(mLastTimeMs); 215 if (DEBUG) Log.d(TAG, "sched @" + mNextScheduledTimeMs + " after " + mLastTimeMs); 216 mTimeProvider.notifyAt( 217 mNextScheduledTimeMs >= 0 ? 218 (mNextScheduledTimeMs * 1000) : MediaTimeProvider.NO_TIME, 219 this); 220 } 221 } 222 223 /** 224 * @hide 225 */ 226 @Override 227 public void onTimedEvent(long timeUs) { 228 if (DEBUG) Log.d(TAG, "onTimedEvent " + timeUs); 229 synchronized (this) { 230 long timeMs = timeUs / 1000; 231 updateActiveCues(false, timeMs); 232 takeTime(timeMs); 233 } 234 updateView(mActiveCues); 235 scheduleTimedEvents(); 236 } 237 238 /** 239 * @hide 240 */ 241 @Override 242 public void onSeek(long timeUs) { 243 if (DEBUG) Log.d(TAG, "onSeek " + timeUs); 244 synchronized (this) { 245 long timeMs = timeUs / 1000; 246 updateActiveCues(true, timeMs); 247 takeTime(timeMs); 248 } 249 updateView(mActiveCues); 250 scheduleTimedEvents(); 251 } 252 253 /** 254 * @hide 255 */ 256 @Override 257 public void onStop() { 258 synchronized (this) { 259 if (DEBUG) Log.d(TAG, "onStop"); 260 clearActiveCues(); 261 mLastTimeMs = -1; 262 } 263 updateView(mActiveCues); 264 mNextScheduledTimeMs = -1; 265 mTimeProvider.notifyAt(MediaTimeProvider.NO_TIME, this); 266 } 267 268 /** @hide */ 269 protected MediaTimeProvider mTimeProvider; 270 271 /** @hide */ 272 public void show() { 273 if (mVisible) { 274 return; 275 } 276 277 mVisible = true; 278 RenderingWidget renderingWidget = getRenderingWidget(); 279 if (renderingWidget != null) { 280 renderingWidget.setVisible(true); 281 } 282 if (mTimeProvider != null) { 283 mTimeProvider.scheduleUpdate(this); 284 } 285 } 286 287 /** @hide */ 288 public void hide() { 289 if (!mVisible) { 290 return; 291 } 292 293 if (mTimeProvider != null) { 294 mTimeProvider.cancelNotifications(this); 295 } 296 RenderingWidget renderingWidget = getRenderingWidget(); 297 if (renderingWidget != null) { 298 renderingWidget.setVisible(false); 299 } 300 mVisible = false; 301 } 302 303 /** @hide */ 304 protected synchronized boolean addCue(Cue cue) { 305 mCues.add(cue); 306 307 if (cue.mRunID != 0) { 308 Run run = mRunsByID.get(cue.mRunID); 309 if (run == null) { 310 run = new Run(); 311 mRunsByID.put(cue.mRunID, run); 312 run.mEndTimeMs = cue.mEndTimeMs; 313 } else if (run.mEndTimeMs < cue.mEndTimeMs) { 314 run.mEndTimeMs = cue.mEndTimeMs; 315 } 316 317 // link-up cues in the same run 318 cue.mNextInRun = run.mFirstCue; 319 run.mFirstCue = cue; 320 } 321 322 // if a cue is added that should be visible, need to refresh view 323 long nowMs = -1; 324 if (mTimeProvider != null) { 325 try { 326 nowMs = mTimeProvider.getCurrentTimeUs( 327 false /* precise */, true /* monotonic */) / 1000; 328 } catch (IllegalStateException e) { 329 // handle as it we are not playing 330 } 331 } 332 333 if (DEBUG) Log.v(TAG, "mVisible=" + mVisible + ", " + 334 cue.mStartTimeMs + " <= " + nowMs + ", " + 335 cue.mEndTimeMs + " >= " + mLastTimeMs); 336 337 if (mVisible && 338 cue.mStartTimeMs <= nowMs && 339 // we don't trust nowMs, so check any cue since last callback 340 cue.mEndTimeMs >= mLastTimeMs) { 341 if (mRunnable != null) { 342 mHandler.removeCallbacks(mRunnable); 343 } 344 final SubtitleTrack track = this; 345 final long thenMs = nowMs; 346 mRunnable = new Runnable() { 347 @Override 348 public void run() { 349 // even with synchronized, it is possible that we are going 350 // to do multiple updates as the runnable could be already 351 // running. 352 synchronized (track) { 353 mRunnable = null; 354 updateActiveCues(true, thenMs); 355 updateView(mActiveCues); 356 } 357 } 358 }; 359 // delay update so we don't update view on every cue. TODO why 10? 360 if (mHandler.postDelayed(mRunnable, 10 /* delay */)) { 361 if (DEBUG) Log.v(TAG, "scheduling update"); 362 } else { 363 if (DEBUG) Log.w(TAG, "failed to schedule subtitle view update"); 364 } 365 return true; 366 } 367 368 if (mVisible && 369 cue.mEndTimeMs >= mLastTimeMs && 370 (cue.mStartTimeMs < mNextScheduledTimeMs || 371 mNextScheduledTimeMs < 0)) { 372 scheduleTimedEvents(); 373 } 374 375 return false; 376 } 377 378 /** @hide */ 379 public synchronized void setTimeProvider(MediaTimeProvider timeProvider) { 380 if (mTimeProvider == timeProvider) { 381 return; 382 } 383 if (mTimeProvider != null) { 384 mTimeProvider.cancelNotifications(this); 385 } 386 mTimeProvider = timeProvider; 387 if (mTimeProvider != null) { 388 mTimeProvider.scheduleUpdate(this); 389 } 390 } 391 392 393 /** @hide */ 394 static class CueList { 395 private static final String TAG = "CueList"; 396 // simplistic, inefficient implementation 397 private SortedMap<Long, Vector<Cue> > mCues; 398 public boolean DEBUG = false; 399 400 private boolean addEvent(Cue cue, long timeMs) { 401 Vector<Cue> cues = mCues.get(timeMs); 402 if (cues == null) { 403 cues = new Vector<Cue>(2); 404 mCues.put(timeMs, cues); 405 } else if (cues.contains(cue)) { 406 // do not duplicate cues 407 return false; 408 } 409 410 cues.add(cue); 411 return true; 412 } 413 414 private void removeEvent(Cue cue, long timeMs) { 415 Vector<Cue> cues = mCues.get(timeMs); 416 if (cues != null) { 417 cues.remove(cue); 418 if (cues.size() == 0) { 419 mCues.remove(timeMs); 420 } 421 } 422 } 423 424 public void add(Cue cue) { 425 // ignore non-positive-duration cues 426 if (cue.mStartTimeMs >= cue.mEndTimeMs) 427 return; 428 429 if (!addEvent(cue, cue.mStartTimeMs)) { 430 return; 431 } 432 433 long lastTimeMs = cue.mStartTimeMs; 434 if (cue.mInnerTimesMs != null) { 435 for (long timeMs: cue.mInnerTimesMs) { 436 if (timeMs > lastTimeMs && timeMs < cue.mEndTimeMs) { 437 addEvent(cue, timeMs); 438 lastTimeMs = timeMs; 439 } 440 } 441 } 442 443 addEvent(cue, cue.mEndTimeMs); 444 } 445 446 public void remove(Cue cue) { 447 removeEvent(cue, cue.mStartTimeMs); 448 if (cue.mInnerTimesMs != null) { 449 for (long timeMs: cue.mInnerTimesMs) { 450 removeEvent(cue, timeMs); 451 } 452 } 453 removeEvent(cue, cue.mEndTimeMs); 454 } 455 456 public Iterable<Pair<Long, Cue>> entriesBetween( 457 final long lastTimeMs, final long timeMs) { 458 return new Iterable<Pair<Long, Cue> >() { 459 @Override 460 public Iterator<Pair<Long, Cue> > iterator() { 461 if (DEBUG) Log.d(TAG, "slice (" + lastTimeMs + ", " + timeMs + "]="); 462 try { 463 return new EntryIterator( 464 mCues.subMap(lastTimeMs + 1, timeMs + 1)); 465 } catch(IllegalArgumentException e) { 466 return new EntryIterator(null); 467 } 468 } 469 }; 470 } 471 472 public long nextTimeAfter(long timeMs) { 473 SortedMap<Long, Vector<Cue>> tail = null; 474 try { 475 tail = mCues.tailMap(timeMs + 1); 476 if (tail != null) { 477 return tail.firstKey(); 478 } else { 479 return -1; 480 } 481 } catch(IllegalArgumentException e) { 482 return -1; 483 } catch(NoSuchElementException e) { 484 return -1; 485 } 486 } 487 488 class EntryIterator implements Iterator<Pair<Long, Cue> > { 489 @Override 490 public boolean hasNext() { 491 return !mDone; 492 } 493 494 @Override 495 public Pair<Long, Cue> next() { 496 if (mDone) { 497 throw new NoSuchElementException(""); 498 } 499 mLastEntry = new Pair<Long, Cue>( 500 mCurrentTimeMs, mListIterator.next()); 501 mLastListIterator = mListIterator; 502 if (!mListIterator.hasNext()) { 503 nextKey(); 504 } 505 return mLastEntry; 506 } 507 508 @Override 509 public void remove() { 510 // only allow removing end tags 511 if (mLastListIterator == null || 512 mLastEntry.second.mEndTimeMs != mLastEntry.first) { 513 throw new IllegalStateException(""); 514 } 515 516 // remove end-cue 517 mLastListIterator.remove(); 518 mLastListIterator = null; 519 if (mCues.get(mLastEntry.first).size() == 0) { 520 mCues.remove(mLastEntry.first); 521 } 522 523 // remove rest of the cues 524 Cue cue = mLastEntry.second; 525 removeEvent(cue, cue.mStartTimeMs); 526 if (cue.mInnerTimesMs != null) { 527 for (long timeMs: cue.mInnerTimesMs) { 528 removeEvent(cue, timeMs); 529 } 530 } 531 } 532 533 public EntryIterator(SortedMap<Long, Vector<Cue> > cues) { 534 if (DEBUG) Log.v(TAG, cues + ""); 535 mRemainingCues = cues; 536 mLastListIterator = null; 537 nextKey(); 538 } 539 540 private void nextKey() { 541 do { 542 try { 543 if (mRemainingCues == null) { 544 throw new NoSuchElementException(""); 545 } 546 mCurrentTimeMs = mRemainingCues.firstKey(); 547 mListIterator = 548 mRemainingCues.get(mCurrentTimeMs).iterator(); 549 try { 550 mRemainingCues = 551 mRemainingCues.tailMap(mCurrentTimeMs + 1); 552 } catch (IllegalArgumentException e) { 553 mRemainingCues = null; 554 } 555 mDone = false; 556 } catch (NoSuchElementException e) { 557 mDone = true; 558 mRemainingCues = null; 559 mListIterator = null; 560 return; 561 } 562 } while (!mListIterator.hasNext()); 563 } 564 565 private long mCurrentTimeMs; 566 private Iterator<Cue> mListIterator; 567 private boolean mDone; 568 private SortedMap<Long, Vector<Cue> > mRemainingCues; 569 private Iterator<Cue> mLastListIterator; 570 private Pair<Long,Cue> mLastEntry; 571 } 572 573 CueList() { 574 mCues = new TreeMap<Long, Vector<Cue>>(); 575 } 576 } 577 578 /** @hide */ 579 public static class Cue { 580 public long mStartTimeMs; 581 public long mEndTimeMs; 582 public long[] mInnerTimesMs; 583 public long mRunID; 584 585 /** @hide */ 586 public Cue mNextInRun; 587 588 public void onTime(long timeMs) { } 589 } 590 591 /** @hide update mRunsByEndTime (with default end time) */ 592 protected void finishedRun(long runID) { 593 if (runID != 0 && runID != ~0) { 594 Run run = mRunsByID.get(runID); 595 if (run != null) { 596 run.storeByEndTimeMs(mRunsByEndTime); 597 } 598 } 599 } 600 601 /** @hide update mRunsByEndTime with given end time */ 602 public void setRunDiscardTimeMs(long runID, long timeMs) { 603 if (runID != 0 && runID != ~0) { 604 Run run = mRunsByID.get(runID); 605 if (run != null) { 606 run.mEndTimeMs = timeMs; 607 run.storeByEndTimeMs(mRunsByEndTime); 608 } 609 } 610 } 611 612 /** @hide whether this is a text track who fires events instead getting rendered */ 613 public int getTrackType() { 614 return getRenderingWidget() == null 615 ? TrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT 616 : TrackInfo.MEDIA_TRACK_TYPE_SUBTITLE; 617 } 618 619 620 /** @hide */ 621 private static class Run { 622 public Cue mFirstCue; 623 public Run mNextRunAtEndTimeMs; 624 public Run mPrevRunAtEndTimeMs; 625 public long mEndTimeMs = -1; 626 public long mRunID = 0; 627 private long mStoredEndTimeMs = -1; 628 629 public void storeByEndTimeMs(LongSparseArray<Run> runsByEndTime) { 630 // remove old value if any 631 int ix = runsByEndTime.indexOfKey(mStoredEndTimeMs); 632 if (ix >= 0) { 633 if (mPrevRunAtEndTimeMs == null) { 634 assert(this == runsByEndTime.valueAt(ix)); 635 if (mNextRunAtEndTimeMs == null) { 636 runsByEndTime.removeAt(ix); 637 } else { 638 runsByEndTime.setValueAt(ix, mNextRunAtEndTimeMs); 639 } 640 } 641 removeAtEndTimeMs(); 642 } 643 644 // add new value 645 if (mEndTimeMs >= 0) { 646 mPrevRunAtEndTimeMs = null; 647 mNextRunAtEndTimeMs = runsByEndTime.get(mEndTimeMs); 648 if (mNextRunAtEndTimeMs != null) { 649 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = this; 650 } 651 runsByEndTime.put(mEndTimeMs, this); 652 mStoredEndTimeMs = mEndTimeMs; 653 } 654 } 655 656 public void removeAtEndTimeMs() { 657 Run prev = mPrevRunAtEndTimeMs; 658 659 if (mPrevRunAtEndTimeMs != null) { 660 mPrevRunAtEndTimeMs.mNextRunAtEndTimeMs = mNextRunAtEndTimeMs; 661 mPrevRunAtEndTimeMs = null; 662 } 663 if (mNextRunAtEndTimeMs != null) { 664 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = prev; 665 mNextRunAtEndTimeMs = null; 666 } 667 } 668 } 669 670 /** 671 * Interface for rendering subtitles onto a Canvas. 672 */ 673 public interface RenderingWidget { 674 /** 675 * Sets the widget's callback, which is used to send updates when the 676 * rendered data has changed. 677 * 678 * @param callback update callback 679 */ 680 public void setOnChangedListener(OnChangedListener callback); 681 682 /** 683 * Sets the widget's size. 684 * 685 * @param width width in pixels 686 * @param height height in pixels 687 */ 688 public void setSize(int width, int height); 689 690 /** 691 * Sets whether the widget should draw subtitles. 692 * 693 * @param visible true if subtitles should be drawn, false otherwise 694 */ 695 public void setVisible(boolean visible); 696 697 /** 698 * Renders subtitles onto a {@link Canvas}. 699 * 700 * @param c canvas on which to render subtitles 701 */ 702 public void draw(Canvas c); 703 704 /** 705 * Called when the widget is attached to a window. 706 */ 707 public void onAttachedToWindow(); 708 709 /** 710 * Called when the widget is detached from a window. 711 */ 712 public void onDetachedFromWindow(); 713 714 /** 715 * Callback used to send updates about changes to rendering data. 716 */ 717 public interface OnChangedListener { 718 /** 719 * Called when the rendering data has changed. 720 * 721 * @param renderingWidget the widget whose data has changed 722 */ 723 public void onChanged(RenderingWidget renderingWidget); 724 } 725 } 726 } 727
