1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "WebmFrameThread" 19 20 #include "WebmConstants.h" 21 #include "WebmFrameThread.h" 22 23 #include <media/stagefright/MetaData.h> 24 #include <media/stagefright/foundation/ADebug.h> 25 26 #include <utils/Log.h> 27 #include <inttypes.h> 28 29 using namespace webm; 30 31 namespace android { 32 33 void *WebmFrameThread::wrap(void *arg) { 34 WebmFrameThread *worker = reinterpret_cast<WebmFrameThread*>(arg); 35 worker->run(); 36 return NULL; 37 } 38 39 status_t WebmFrameThread::start() { 40 pthread_attr_t attr; 41 pthread_attr_init(&attr); 42 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); 43 pthread_create(&mThread, &attr, WebmFrameThread::wrap, this); 44 pthread_attr_destroy(&attr); 45 return OK; 46 } 47 48 status_t WebmFrameThread::stop() { 49 void *status; 50 pthread_join(mThread, &status); 51 return (status_t)(intptr_t)status; 52 } 53 54 //================================================================================================= 55 56 WebmFrameSourceThread::WebmFrameSourceThread( 57 int type, 58 LinkedBlockingQueue<const sp<WebmFrame> >& sink) 59 : mType(type), mSink(sink) { 60 } 61 62 //================================================================================================= 63 64 WebmFrameSinkThread::WebmFrameSinkThread( 65 const int& fd, 66 const uint64_t& off, 67 sp<WebmFrameSourceThread> videoThread, 68 sp<WebmFrameSourceThread> audioThread, 69 List<sp<WebmElement> >& cues) 70 : mFd(fd), 71 mSegmentDataStart(off), 72 mVideoFrames(videoThread->mSink), 73 mAudioFrames(audioThread->mSink), 74 mCues(cues), 75 mDone(true) { 76 } 77 78 WebmFrameSinkThread::WebmFrameSinkThread( 79 const int& fd, 80 const uint64_t& off, 81 LinkedBlockingQueue<const sp<WebmFrame> >& videoSource, 82 LinkedBlockingQueue<const sp<WebmFrame> >& audioSource, 83 List<sp<WebmElement> >& cues) 84 : mFd(fd), 85 mSegmentDataStart(off), 86 mVideoFrames(videoSource), 87 mAudioFrames(audioSource), 88 mCues(cues), 89 mDone(true) { 90 } 91 92 // Initializes a webm cluster with its starting timecode. 93 // 94 // frames: 95 // sequence of input audio/video frames received from the source. 96 // 97 // clusterTimecodeL: 98 // the starting timecode of the cluster; this is the timecode of the first 99 // frame since frames are ordered by timestamp. 100 // 101 // children: 102 // list to hold child elements in a webm cluster (start timecode and 103 // simple blocks). 104 // 105 // static 106 void WebmFrameSinkThread::initCluster( 107 List<const sp<WebmFrame> >& frames, 108 uint64_t& clusterTimecodeL, 109 List<sp<WebmElement> >& children) { 110 CHECK(!frames.empty() && children.empty()); 111 112 const sp<WebmFrame> f = *(frames.begin()); 113 clusterTimecodeL = f->mAbsTimecode; 114 WebmUnsigned *clusterTimecode = new WebmUnsigned(kMkvTimecode, clusterTimecodeL); 115 children.clear(); 116 children.push_back(clusterTimecode); 117 } 118 119 void WebmFrameSinkThread::writeCluster(List<sp<WebmElement> >& children) { 120 // children must contain at least one simpleblock and its timecode 121 CHECK_GE(children.size(), 2); 122 123 uint64_t size; 124 sp<WebmElement> cluster = new WebmMaster(kMkvCluster, children); 125 cluster->write(mFd, size); 126 children.clear(); 127 } 128 129 // Write out (possibly multiple) webm cluster(s) from frames split on video key frames. 130 // 131 // last: 132 // current flush is triggered by EOS instead of a second outstanding video key frame. 133 void WebmFrameSinkThread::flushFrames(List<const sp<WebmFrame> >& frames, bool last) { 134 if (frames.empty()) { 135 return; 136 } 137 138 uint64_t clusterTimecodeL; 139 List<sp<WebmElement> > children; 140 initCluster(frames, clusterTimecodeL, children); 141 142 uint64_t cueTime = clusterTimecodeL; 143 off_t fpos = ::lseek(mFd, 0, SEEK_CUR); 144 size_t n = frames.size(); 145 if (!last) { 146 // If we are not flushing the last sequence of outstanding frames, flushFrames 147 // must have been called right after we have pushed a second outstanding video key 148 // frame (the last frame), which belongs to the next cluster; also hold back on 149 // flushing the second to last frame before we check its type. A audio frame 150 // should precede the aforementioned video key frame in the next sequence, a video 151 // frame should be the last frame in the current (to-be-flushed) sequence. 152 CHECK_GE(n, 2); 153 n -= 2; 154 } 155 156 for (size_t i = 0; i < n; i++) { 157 const sp<WebmFrame> f = *(frames.begin()); 158 if (f->mType == kVideoType && f->mKey) { 159 cueTime = f->mAbsTimecode; 160 } 161 162 if (f->mAbsTimecode - clusterTimecodeL > INT16_MAX) { 163 writeCluster(children); 164 initCluster(frames, clusterTimecodeL, children); 165 } 166 167 frames.erase(frames.begin()); 168 children.push_back(f->SimpleBlock(clusterTimecodeL)); 169 } 170 171 // equivalent to last==false 172 if (!frames.empty()) { 173 // decide whether to write out the second to last frame. 174 const sp<WebmFrame> secondLastFrame = *(frames.begin()); 175 if (secondLastFrame->mType == kVideoType) { 176 frames.erase(frames.begin()); 177 children.push_back(secondLastFrame->SimpleBlock(clusterTimecodeL)); 178 } 179 } 180 181 writeCluster(children); 182 sp<WebmElement> cuePoint = WebmElement::CuePointEntry(cueTime, 1, fpos - mSegmentDataStart); 183 mCues.push_back(cuePoint); 184 } 185 186 status_t WebmFrameSinkThread::start() { 187 mDone = false; 188 return WebmFrameThread::start(); 189 } 190 191 status_t WebmFrameSinkThread::stop() { 192 mDone = true; 193 mVideoFrames.push(WebmFrame::EOS); 194 mAudioFrames.push(WebmFrame::EOS); 195 return WebmFrameThread::stop(); 196 } 197 198 void WebmFrameSinkThread::run() { 199 int numVideoKeyFrames = 0; 200 List<const sp<WebmFrame> > outstandingFrames; 201 while (!mDone) { 202 ALOGV("wait v frame"); 203 const sp<WebmFrame> videoFrame = mVideoFrames.peek(); 204 ALOGV("v frame: %p", videoFrame.get()); 205 206 ALOGV("wait a frame"); 207 const sp<WebmFrame> audioFrame = mAudioFrames.peek(); 208 ALOGV("a frame: %p", audioFrame.get()); 209 210 if (videoFrame->mEos && audioFrame->mEos) { 211 break; 212 } 213 214 if (*audioFrame < *videoFrame) { 215 ALOGV("take a frame"); 216 mAudioFrames.take(); 217 outstandingFrames.push_back(audioFrame); 218 } else { 219 ALOGV("take v frame"); 220 mVideoFrames.take(); 221 outstandingFrames.push_back(videoFrame); 222 if (videoFrame->mKey) 223 numVideoKeyFrames++; 224 } 225 226 if (numVideoKeyFrames == 2) { 227 flushFrames(outstandingFrames, /* last = */ false); 228 numVideoKeyFrames--; 229 } 230 } 231 ALOGV("flushing last cluster (size %zu)", outstandingFrames.size()); 232 flushFrames(outstandingFrames, /* last = */ true); 233 mDone = true; 234 } 235 236 //================================================================================================= 237 238 static const int64_t kInitialDelayTimeUs = 700000LL; 239 240 void WebmFrameMediaSourceThread::clearFlags() { 241 mDone = false; 242 mPaused = false; 243 mResumed = false; 244 mStarted = false; 245 mReachedEOS = false; 246 } 247 248 WebmFrameMediaSourceThread::WebmFrameMediaSourceThread( 249 const sp<MediaSource>& source, 250 int type, 251 LinkedBlockingQueue<const sp<WebmFrame> >& sink, 252 uint64_t timeCodeScale, 253 int64_t startTimeRealUs, 254 int32_t startTimeOffsetMs, 255 int numTracks, 256 bool realTimeRecording) 257 : WebmFrameSourceThread(type, sink), 258 mSource(source), 259 mTimeCodeScale(timeCodeScale), 260 mTrackDurationUs(0) { 261 clearFlags(); 262 mStartTimeUs = startTimeRealUs; 263 if (realTimeRecording && numTracks > 1) { 264 /* 265 * Copied from MPEG4Writer 266 * 267 * This extra delay of accepting incoming audio/video signals 268 * helps to align a/v start time at the beginning of a recording 269 * session, and it also helps eliminate the "recording" sound for 270 * camcorder applications. 271 * 272 * If client does not set the start time offset, we fall back to 273 * use the default initial delay value. 274 */ 275 int64_t startTimeOffsetUs = startTimeOffsetMs * 1000LL; 276 if (startTimeOffsetUs < 0) { // Start time offset was not set 277 startTimeOffsetUs = kInitialDelayTimeUs; 278 } 279 mStartTimeUs += startTimeOffsetUs; 280 ALOGI("Start time offset: %" PRId64 " us", startTimeOffsetUs); 281 } 282 } 283 284 status_t WebmFrameMediaSourceThread::start() { 285 sp<MetaData> meta = new MetaData; 286 meta->setInt64(kKeyTime, mStartTimeUs); 287 status_t err = mSource->start(meta.get()); 288 if (err != OK) { 289 mDone = true; 290 mReachedEOS = true; 291 return err; 292 } else { 293 mStarted = true; 294 return WebmFrameThread::start(); 295 } 296 } 297 298 status_t WebmFrameMediaSourceThread::resume() { 299 if (!mDone && mPaused) { 300 mPaused = false; 301 mResumed = true; 302 } 303 return OK; 304 } 305 306 status_t WebmFrameMediaSourceThread::pause() { 307 if (mStarted) { 308 mPaused = true; 309 } 310 return OK; 311 } 312 313 status_t WebmFrameMediaSourceThread::stop() { 314 if (mStarted) { 315 mStarted = false; 316 mDone = true; 317 mSource->stop(); 318 return WebmFrameThread::stop(); 319 } 320 return OK; 321 } 322 323 void WebmFrameMediaSourceThread::run() { 324 int32_t count = 0; 325 int64_t timestampUs = 0xdeadbeef; 326 int64_t lastTimestampUs = 0; // Previous sample time stamp 327 int64_t lastDurationUs = 0; // Previous sample duration 328 int64_t previousPausedDurationUs = 0; 329 330 const uint64_t kUninitialized = 0xffffffffffffffffL; 331 mStartTimeUs = kUninitialized; 332 333 status_t err = OK; 334 MediaBuffer *buffer; 335 while (!mDone && (err = mSource->read(&buffer, NULL)) == OK) { 336 if (buffer->range_length() == 0) { 337 buffer->release(); 338 buffer = NULL; 339 continue; 340 } 341 342 sp<MetaData> md = buffer->meta_data(); 343 CHECK(md->findInt64(kKeyTime, ×tampUs)); 344 if (mStartTimeUs == kUninitialized) { 345 mStartTimeUs = timestampUs; 346 } 347 timestampUs -= mStartTimeUs; 348 349 if (mPaused && !mResumed) { 350 lastDurationUs = timestampUs - lastTimestampUs; 351 lastTimestampUs = timestampUs; 352 buffer->release(); 353 buffer = NULL; 354 continue; 355 } 356 ++count; 357 358 // adjust time-stamps after pause/resume 359 if (mResumed) { 360 int64_t durExcludingEarlierPausesUs = timestampUs - previousPausedDurationUs; 361 CHECK_GE(durExcludingEarlierPausesUs, 0ll); 362 int64_t pausedDurationUs = durExcludingEarlierPausesUs - mTrackDurationUs; 363 CHECK_GE(pausedDurationUs, lastDurationUs); 364 previousPausedDurationUs += pausedDurationUs - lastDurationUs; 365 mResumed = false; 366 } 367 timestampUs -= previousPausedDurationUs; 368 CHECK_GE(timestampUs, 0ll); 369 370 int32_t isSync = false; 371 md->findInt32(kKeyIsSyncFrame, &isSync); 372 const sp<WebmFrame> f = new WebmFrame( 373 mType, 374 isSync, 375 timestampUs * 1000 / mTimeCodeScale, 376 buffer); 377 mSink.push(f); 378 379 ALOGV( 380 "%s %s frame at %" PRId64 " size %zu\n", 381 mType == kVideoType ? "video" : "audio", 382 isSync ? "I" : "P", 383 timestampUs * 1000 / mTimeCodeScale, 384 buffer->range_length()); 385 386 buffer->release(); 387 buffer = NULL; 388 389 if (timestampUs > mTrackDurationUs) { 390 mTrackDurationUs = timestampUs; 391 } 392 lastDurationUs = timestampUs - lastTimestampUs; 393 lastTimestampUs = timestampUs; 394 } 395 396 mTrackDurationUs += lastDurationUs; 397 mSink.push(WebmFrame::EOS); 398 } 399 } 400