1 /* 2 * Copyright 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.media.subtitle; 18 19 import android.graphics.Canvas; 20 import android.media.MediaFormat; 21 import android.media.MediaPlayer2.TrackInfo; 22 import android.media.SubtitleData; 23 import android.os.Handler; 24 import android.util.Log; 25 import android.util.LongSparseArray; 26 import android.util.Pair; 27 28 import java.util.Iterator; 29 import java.util.NoSuchElementException; 30 import java.util.SortedMap; 31 import java.util.TreeMap; 32 import java.util.Vector; 33 34 // Note: This is forked from android.media.SubtitleTrack since P 35 /** 36 * A subtitle track abstract base class that is responsible for parsing and displaying 37 * an instance of a particular type of subtitle. 38 */ 39 public abstract class SubtitleTrack implements MediaTimeProvider.OnMediaTimeListener { 40 private static final String TAG = "SubtitleTrack"; 41 private long mLastUpdateTimeMs; 42 private long mLastTimeMs; 43 44 private Runnable mRunnable; 45 46 final private LongSparseArray<Run> mRunsByEndTime = new LongSparseArray<Run>(); 47 final private LongSparseArray<Run> mRunsByID = new LongSparseArray<Run>(); 48 49 private CueList mCues; 50 final private Vector<Cue> mActiveCues = new Vector<Cue>(); 51 protected boolean mVisible; 52 53 public boolean DEBUG = false; 54 55 protected Handler mHandler = new Handler(); 56 57 private MediaFormat mFormat; 58 59 public SubtitleTrack(MediaFormat format) { 60 mFormat = format; 61 mCues = new CueList(); 62 clearActiveCues(); 63 mLastTimeMs = -1; 64 } 65 66 public final MediaFormat getFormat() { 67 return mFormat; 68 } 69 70 private long mNextScheduledTimeMs = -1; 71 72 public void onData(SubtitleData data) { 73 long runID = data.getStartTimeUs() + 1; 74 onData(data.getData(), true /* eos */, runID); 75 setRunDiscardTimeMs( 76 runID, 77 (data.getStartTimeUs() + data.getDurationUs()) / 1000); 78 } 79 80 /** 81 * Called when there is input data for the subtitle track. The 82 * complete subtitle for a track can include multiple whole units 83 * (runs). Each of these units can have multiple sections. The 84 * contents of a run are submitted in sequential order, with eos 85 * indicating the last section of the run. Calls from different 86 * runs must not be intermixed. 87 * 88 * @param data subtitle data byte buffer 89 * @param eos true if this is the last section of the run. 90 * @param runID mostly-unique ID for this run of data. Subtitle cues 91 * with runID of 0 are discarded immediately after 92 * display. Cues with runID of ~0 are discarded 93 * only at the deletion of the track object. Cues 94 * with other runID-s are discarded at the end of the 95 * run, which defaults to the latest timestamp of 96 * any of its cues (with this runID). 97 */ 98 protected abstract void onData(byte[] data, boolean eos, long runID); 99 100 /** 101 * Called when adding the subtitle rendering widget to the view hierarchy, 102 * as well as when showing or hiding the subtitle track, or when the video 103 * surface position has changed. 104 * 105 * @return the widget that renders this subtitle track. For most renderers 106 * there should be a single shared instance that is used for all 107 * tracks supported by that renderer, as at most one subtitle track 108 * is visible at one time. 109 */ 110 public abstract RenderingWidget getRenderingWidget(); 111 112 /** 113 * Called when the active cues have changed, and the contents of the subtitle 114 * view should be updated. 115 */ 116 public abstract void updateView(Vector<Cue> activeCues); 117 118 protected synchronized void updateActiveCues(boolean rebuild, long timeMs) { 119 // out-of-order times mean seeking or new active cues being added 120 // (during their own timespan) 121 if (rebuild || mLastUpdateTimeMs > timeMs) { 122 clearActiveCues(); 123 } 124 125 for(Iterator<Pair<Long, Cue> > it = 126 mCues.entriesBetween(mLastUpdateTimeMs, timeMs).iterator(); it.hasNext(); ) { 127 Pair<Long, Cue> event = it.next(); 128 Cue cue = event.second; 129 130 if (cue.mEndTimeMs == event.first) { 131 // remove past cues 132 if (DEBUG) Log.v(TAG, "Removing " + cue); 133 mActiveCues.remove(cue); 134 if (cue.mRunID == 0) { 135 it.remove(); 136 } 137 } else if (cue.mStartTimeMs == event.first) { 138 // add new cues 139 // TRICKY: this will happen in start order 140 if (DEBUG) Log.v(TAG, "Adding " + cue); 141 if (cue.mInnerTimesMs != null) { 142 cue.onTime(timeMs); 143 } 144 mActiveCues.add(cue); 145 } else if (cue.mInnerTimesMs != null) { 146 // cue is modified 147 cue.onTime(timeMs); 148 } 149 } 150 151 /* complete any runs */ 152 while (mRunsByEndTime.size() > 0 && 153 mRunsByEndTime.keyAt(0) <= timeMs) { 154 removeRunsByEndTimeIndex(0); // removes element 155 } 156 mLastUpdateTimeMs = timeMs; 157 } 158 159 private void removeRunsByEndTimeIndex(int ix) { 160 Run run = mRunsByEndTime.valueAt(ix); 161 while (run != null) { 162 Cue cue = run.mFirstCue; 163 while (cue != null) { 164 mCues.remove(cue); 165 Cue nextCue = cue.mNextInRun; 166 cue.mNextInRun = null; 167 cue = nextCue; 168 } 169 mRunsByID.remove(run.mRunID); 170 Run nextRun = run.mNextRunAtEndTimeMs; 171 run.mPrevRunAtEndTimeMs = null; 172 run.mNextRunAtEndTimeMs = null; 173 run = nextRun; 174 } 175 mRunsByEndTime.removeAt(ix); 176 } 177 178 @Override 179 protected void finalize() throws Throwable { 180 /* remove all cues (untangle all cross-links) */ 181 int size = mRunsByEndTime.size(); 182 for(int ix = size - 1; ix >= 0; ix--) { 183 removeRunsByEndTimeIndex(ix); 184 } 185 186 super.finalize(); 187 } 188 189 private synchronized void takeTime(long timeMs) { 190 mLastTimeMs = timeMs; 191 } 192 193 protected synchronized void clearActiveCues() { 194 if (DEBUG) Log.v(TAG, "Clearing " + mActiveCues.size() + " active cues"); 195 mActiveCues.clear(); 196 mLastUpdateTimeMs = -1; 197 } 198 199 protected void scheduleTimedEvents() { 200 /* get times for the next event */ 201 if (mTimeProvider != null) { 202 mNextScheduledTimeMs = mCues.nextTimeAfter(mLastTimeMs); 203 if (DEBUG) Log.d(TAG, "sched @" + mNextScheduledTimeMs + " after " + mLastTimeMs); 204 mTimeProvider.notifyAt( 205 mNextScheduledTimeMs >= 0 ? 206 (mNextScheduledTimeMs * 1000) : MediaTimeProvider.NO_TIME, 207 this); 208 } 209 } 210 211 @Override 212 public void onTimedEvent(long timeUs) { 213 if (DEBUG) Log.d(TAG, "onTimedEvent " + timeUs); 214 synchronized (this) { 215 long timeMs = timeUs / 1000; 216 updateActiveCues(false, timeMs); 217 takeTime(timeMs); 218 } 219 updateView(mActiveCues); 220 scheduleTimedEvents(); 221 } 222 223 @Override 224 public void onSeek(long timeUs) { 225 if (DEBUG) Log.d(TAG, "onSeek " + timeUs); 226 synchronized (this) { 227 long timeMs = timeUs / 1000; 228 updateActiveCues(true, timeMs); 229 takeTime(timeMs); 230 } 231 updateView(mActiveCues); 232 scheduleTimedEvents(); 233 } 234 235 @Override 236 public void onStop() { 237 synchronized (this) { 238 if (DEBUG) Log.d(TAG, "onStop"); 239 clearActiveCues(); 240 mLastTimeMs = -1; 241 } 242 updateView(mActiveCues); 243 mNextScheduledTimeMs = -1; 244 mTimeProvider.notifyAt(MediaTimeProvider.NO_TIME, this); 245 } 246 247 protected MediaTimeProvider mTimeProvider; 248 249 public void show() { 250 if (mVisible) { 251 return; 252 } 253 254 mVisible = true; 255 RenderingWidget renderingWidget = getRenderingWidget(); 256 if (renderingWidget != null) { 257 renderingWidget.setVisible(true); 258 } 259 if (mTimeProvider != null) { 260 mTimeProvider.scheduleUpdate(this); 261 } 262 } 263 264 public void hide() { 265 if (!mVisible) { 266 return; 267 } 268 269 if (mTimeProvider != null) { 270 mTimeProvider.cancelNotifications(this); 271 } 272 RenderingWidget renderingWidget = getRenderingWidget(); 273 if (renderingWidget != null) { 274 renderingWidget.setVisible(false); 275 } 276 mVisible = false; 277 } 278 279 protected synchronized boolean addCue(Cue cue) { 280 mCues.add(cue); 281 282 if (cue.mRunID != 0) { 283 Run run = mRunsByID.get(cue.mRunID); 284 if (run == null) { 285 run = new Run(); 286 mRunsByID.put(cue.mRunID, run); 287 run.mEndTimeMs = cue.mEndTimeMs; 288 } else if (run.mEndTimeMs < cue.mEndTimeMs) { 289 run.mEndTimeMs = cue.mEndTimeMs; 290 } 291 292 // link-up cues in the same run 293 cue.mNextInRun = run.mFirstCue; 294 run.mFirstCue = cue; 295 } 296 297 // if a cue is added that should be visible, need to refresh view 298 long nowMs = -1; 299 if (mTimeProvider != null) { 300 try { 301 nowMs = mTimeProvider.getCurrentTimeUs( 302 false /* precise */, true /* monotonic */) / 1000; 303 } catch (IllegalStateException e) { 304 // handle as it we are not playing 305 } 306 } 307 308 if (DEBUG) Log.v(TAG, "mVisible=" + mVisible + ", " + 309 cue.mStartTimeMs + " <= " + nowMs + ", " + 310 cue.mEndTimeMs + " >= " + mLastTimeMs); 311 312 if (mVisible && 313 cue.mStartTimeMs <= nowMs && 314 // we don't trust nowMs, so check any cue since last callback 315 cue.mEndTimeMs >= mLastTimeMs) { 316 if (mRunnable != null) { 317 mHandler.removeCallbacks(mRunnable); 318 } 319 final SubtitleTrack track = this; 320 final long thenMs = nowMs; 321 mRunnable = new Runnable() { 322 @Override 323 public void run() { 324 // even with synchronized, it is possible that we are going 325 // to do multiple updates as the runnable could be already 326 // running. 327 synchronized (track) { 328 mRunnable = null; 329 updateActiveCues(true, thenMs); 330 updateView(mActiveCues); 331 } 332 } 333 }; 334 // delay update so we don't update view on every cue. TODO why 10? 335 if (mHandler.postDelayed(mRunnable, 10 /* delay */)) { 336 if (DEBUG) Log.v(TAG, "scheduling update"); 337 } else { 338 if (DEBUG) Log.w(TAG, "failed to schedule subtitle view update"); 339 } 340 return true; 341 } 342 343 if (mVisible && 344 cue.mEndTimeMs >= mLastTimeMs && 345 (cue.mStartTimeMs < mNextScheduledTimeMs || 346 mNextScheduledTimeMs < 0)) { 347 scheduleTimedEvents(); 348 } 349 350 return false; 351 } 352 353 public synchronized void setTimeProvider(MediaTimeProvider timeProvider) { 354 if (mTimeProvider == timeProvider) { 355 return; 356 } 357 if (mTimeProvider != null) { 358 mTimeProvider.cancelNotifications(this); 359 } 360 mTimeProvider = timeProvider; 361 if (mTimeProvider != null) { 362 mTimeProvider.scheduleUpdate(this); 363 } 364 } 365 366 367 static class CueList { 368 private static final String TAG = "CueList"; 369 // simplistic, inefficient implementation 370 private SortedMap<Long, Vector<Cue> > mCues; 371 public boolean DEBUG = false; 372 373 private boolean addEvent(Cue cue, long timeMs) { 374 Vector<Cue> cues = mCues.get(timeMs); 375 if (cues == null) { 376 cues = new Vector<Cue>(2); 377 mCues.put(timeMs, cues); 378 } else if (cues.contains(cue)) { 379 // do not duplicate cues 380 return false; 381 } 382 383 cues.add(cue); 384 return true; 385 } 386 387 private void removeEvent(Cue cue, long timeMs) { 388 Vector<Cue> cues = mCues.get(timeMs); 389 if (cues != null) { 390 cues.remove(cue); 391 if (cues.size() == 0) { 392 mCues.remove(timeMs); 393 } 394 } 395 } 396 397 public void add(Cue cue) { 398 // ignore non-positive-duration cues 399 if (cue.mStartTimeMs >= cue.mEndTimeMs) 400 return; 401 402 if (!addEvent(cue, cue.mStartTimeMs)) { 403 return; 404 } 405 406 long lastTimeMs = cue.mStartTimeMs; 407 if (cue.mInnerTimesMs != null) { 408 for (long timeMs: cue.mInnerTimesMs) { 409 if (timeMs > lastTimeMs && timeMs < cue.mEndTimeMs) { 410 addEvent(cue, timeMs); 411 lastTimeMs = timeMs; 412 } 413 } 414 } 415 416 addEvent(cue, cue.mEndTimeMs); 417 } 418 419 public void remove(Cue cue) { 420 removeEvent(cue, cue.mStartTimeMs); 421 if (cue.mInnerTimesMs != null) { 422 for (long timeMs: cue.mInnerTimesMs) { 423 removeEvent(cue, timeMs); 424 } 425 } 426 removeEvent(cue, cue.mEndTimeMs); 427 } 428 429 public Iterable<Pair<Long, Cue>> entriesBetween( 430 final long lastTimeMs, final long timeMs) { 431 return new Iterable<Pair<Long, Cue> >() { 432 @Override 433 public Iterator<Pair<Long, Cue> > iterator() { 434 if (DEBUG) Log.d(TAG, "slice (" + lastTimeMs + ", " + timeMs + "]="); 435 try { 436 return new EntryIterator( 437 mCues.subMap(lastTimeMs + 1, timeMs + 1)); 438 } catch(IllegalArgumentException e) { 439 return new EntryIterator(null); 440 } 441 } 442 }; 443 } 444 445 public long nextTimeAfter(long timeMs) { 446 SortedMap<Long, Vector<Cue>> tail = null; 447 try { 448 tail = mCues.tailMap(timeMs + 1); 449 if (tail != null) { 450 return tail.firstKey(); 451 } else { 452 return -1; 453 } 454 } catch(IllegalArgumentException e) { 455 return -1; 456 } catch(NoSuchElementException e) { 457 return -1; 458 } 459 } 460 461 class EntryIterator implements Iterator<Pair<Long, Cue> > { 462 @Override 463 public boolean hasNext() { 464 return !mDone; 465 } 466 467 @Override 468 public Pair<Long, Cue> next() { 469 if (mDone) { 470 throw new NoSuchElementException(""); 471 } 472 mLastEntry = new Pair<Long, Cue>( 473 mCurrentTimeMs, mListIterator.next()); 474 mLastListIterator = mListIterator; 475 if (!mListIterator.hasNext()) { 476 nextKey(); 477 } 478 return mLastEntry; 479 } 480 481 @Override 482 public void remove() { 483 // only allow removing end tags 484 if (mLastListIterator == null || 485 mLastEntry.second.mEndTimeMs != mLastEntry.first) { 486 throw new IllegalStateException(""); 487 } 488 489 // remove end-cue 490 mLastListIterator.remove(); 491 mLastListIterator = null; 492 if (mCues.get(mLastEntry.first).size() == 0) { 493 mCues.remove(mLastEntry.first); 494 } 495 496 // remove rest of the cues 497 Cue cue = mLastEntry.second; 498 removeEvent(cue, cue.mStartTimeMs); 499 if (cue.mInnerTimesMs != null) { 500 for (long timeMs: cue.mInnerTimesMs) { 501 removeEvent(cue, timeMs); 502 } 503 } 504 } 505 506 public EntryIterator(SortedMap<Long, Vector<Cue> > cues) { 507 if (DEBUG) Log.v(TAG, cues + ""); 508 mRemainingCues = cues; 509 mLastListIterator = null; 510 nextKey(); 511 } 512 513 private void nextKey() { 514 do { 515 try { 516 if (mRemainingCues == null) { 517 throw new NoSuchElementException(""); 518 } 519 mCurrentTimeMs = mRemainingCues.firstKey(); 520 mListIterator = 521 mRemainingCues.get(mCurrentTimeMs).iterator(); 522 try { 523 mRemainingCues = 524 mRemainingCues.tailMap(mCurrentTimeMs + 1); 525 } catch (IllegalArgumentException e) { 526 mRemainingCues = null; 527 } 528 mDone = false; 529 } catch (NoSuchElementException e) { 530 mDone = true; 531 mRemainingCues = null; 532 mListIterator = null; 533 return; 534 } 535 } while (!mListIterator.hasNext()); 536 } 537 538 private long mCurrentTimeMs; 539 private Iterator<Cue> mListIterator; 540 private boolean mDone; 541 private SortedMap<Long, Vector<Cue> > mRemainingCues; 542 private Iterator<Cue> mLastListIterator; 543 private Pair<Long,Cue> mLastEntry; 544 } 545 546 CueList() { 547 mCues = new TreeMap<Long, Vector<Cue>>(); 548 } 549 } 550 551 public static class Cue { 552 public long mStartTimeMs; 553 public long mEndTimeMs; 554 public long[] mInnerTimesMs; 555 public long mRunID; 556 557 public Cue mNextInRun; 558 559 public void onTime(long timeMs) { } 560 } 561 562 /** update mRunsByEndTime (with default end time) */ 563 protected void finishedRun(long runID) { 564 if (runID != 0 && runID != ~0) { 565 Run run = mRunsByID.get(runID); 566 if (run != null) { 567 run.storeByEndTimeMs(mRunsByEndTime); 568 } 569 } 570 } 571 572 /** update mRunsByEndTime with given end time */ 573 public void setRunDiscardTimeMs(long runID, long timeMs) { 574 if (runID != 0 && runID != ~0) { 575 Run run = mRunsByID.get(runID); 576 if (run != null) { 577 run.mEndTimeMs = timeMs; 578 run.storeByEndTimeMs(mRunsByEndTime); 579 } 580 } 581 } 582 583 /** whether this is a text track who fires events instead getting rendered */ 584 public int getTrackType() { 585 return getRenderingWidget() == null 586 ? TrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT 587 : TrackInfo.MEDIA_TRACK_TYPE_SUBTITLE; 588 } 589 590 591 private static class Run { 592 public Cue mFirstCue; 593 public Run mNextRunAtEndTimeMs; 594 public Run mPrevRunAtEndTimeMs; 595 public long mEndTimeMs = -1; 596 public long mRunID = 0; 597 private long mStoredEndTimeMs = -1; 598 599 public void storeByEndTimeMs(LongSparseArray<Run> runsByEndTime) { 600 // remove old value if any 601 int ix = runsByEndTime.indexOfKey(mStoredEndTimeMs); 602 if (ix >= 0) { 603 if (mPrevRunAtEndTimeMs == null) { 604 assert(this == runsByEndTime.valueAt(ix)); 605 if (mNextRunAtEndTimeMs == null) { 606 runsByEndTime.removeAt(ix); 607 } else { 608 runsByEndTime.setValueAt(ix, mNextRunAtEndTimeMs); 609 } 610 } 611 removeAtEndTimeMs(); 612 } 613 614 // add new value 615 if (mEndTimeMs >= 0) { 616 mPrevRunAtEndTimeMs = null; 617 mNextRunAtEndTimeMs = runsByEndTime.get(mEndTimeMs); 618 if (mNextRunAtEndTimeMs != null) { 619 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = this; 620 } 621 runsByEndTime.put(mEndTimeMs, this); 622 mStoredEndTimeMs = mEndTimeMs; 623 } 624 } 625 626 public void removeAtEndTimeMs() { 627 Run prev = mPrevRunAtEndTimeMs; 628 629 if (mPrevRunAtEndTimeMs != null) { 630 mPrevRunAtEndTimeMs.mNextRunAtEndTimeMs = mNextRunAtEndTimeMs; 631 mPrevRunAtEndTimeMs = null; 632 } 633 if (mNextRunAtEndTimeMs != null) { 634 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = prev; 635 mNextRunAtEndTimeMs = null; 636 } 637 } 638 } 639 640 /** 641 * Interface for rendering subtitles onto a Canvas. 642 */ 643 public interface RenderingWidget { 644 /** 645 * Sets the widget's callback, which is used to send updates when the 646 * rendered data has changed. 647 * 648 * @param callback update callback 649 */ 650 public void setOnChangedListener(OnChangedListener callback); 651 652 /** 653 * Sets the widget's size. 654 * 655 * @param width width in pixels 656 * @param height height in pixels 657 */ 658 public void setSize(int width, int height); 659 660 /** 661 * Sets whether the widget should draw subtitles. 662 * 663 * @param visible true if subtitles should be drawn, false otherwise 664 */ 665 public void setVisible(boolean visible); 666 667 /** 668 * Renders subtitles onto a {@link Canvas}. 669 * 670 * @param c canvas on which to render subtitles 671 */ 672 public void draw(Canvas c); 673 674 /** 675 * Called when the widget is attached to a window. 676 */ 677 public void onAttachedToWindow(); 678 679 /** 680 * Called when the widget is detached from a window. 681 */ 682 public void onDetachedFromWindow(); 683 684 /** 685 * Callback used to send updates about changes to rendering data. 686 */ 687 public interface OnChangedListener { 688 /** 689 * Called when the rendering data has changed. 690 * 691 * @param renderingWidget the widget whose data has changed 692 */ 693 public void onChanged(RenderingWidget renderingWidget); 694 } 695 } 696 } 697