Home | History | Annotate | Download | only in jni 
      1 /*
      2  * Copyright 2015, The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "MediaSync-JNI"
     19 #include <utils/Log.h>
     20 
     21 #include "android_media_MediaSync.h"
     22 
     23 #include "android_media_AudioTrack.h"
     24 #include "android_media_PlaybackParams.h"
     25 #include "android_media_SyncParams.h"
     26 #include "android_runtime/AndroidRuntime.h"
     27 #include "android_runtime/android_view_Surface.h"
     28 #include "jni.h"
     29 #include <nativehelper/JNIHelp.h>
     30 
     31 #include <gui/Surface.h>
     32 
     33 #include <media/AudioResamplerPublic.h>
     34 #include <media/AudioTrack.h>
     35 #include <media/stagefright/MediaClock.h>
     36 #include <media/stagefright/MediaSync.h>
     37 #include <media/stagefright/foundation/ADebug.h>
     38 #include <media/stagefright/foundation/AString.h>
     39 
     40 #include <nativehelper/ScopedLocalRef.h>
     41 
     42 namespace android {
     43 
     44 struct fields_t {
     45     jfieldID context;
     46     jfieldID mediaTimestampMediaTimeUsID;
     47     jfieldID mediaTimestampNanoTimeID;
     48     jfieldID mediaTimestampClockRateID;
     49 };
     50 
     51 static fields_t gFields;
     52 static PlaybackParams::fields_t gPlaybackParamsFields;
     53 static SyncParams::fields_t gSyncParamsFields;
     54 
     55 ////////////////////////////////////////////////////////////////////////////////
     56 
     57 JMediaSync::JMediaSync() {
     58     mSync = MediaSync::create();
     59 }
     60 
     61 JMediaSync::~JMediaSync() {
     62 }
     63 
     64 status_t JMediaSync::setSurface(const sp<IGraphicBufferProducer> &bufferProducer) {
     65     return mSync->setSurface(bufferProducer);
     66 }
     67 
     68 status_t JMediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) {
     69     return mSync->setAudioTrack(audioTrack);
     70 }
     71 
     72 status_t JMediaSync::createInputSurface(
     73         sp<IGraphicBufferProducer>* bufferProducer) {
     74     return mSync->createInputSurface(bufferProducer);
     75 }
     76 
     77 sp<const MediaClock> JMediaSync::getMediaClock() {
     78     return mSync->getMediaClock();
     79 }
     80 
     81 status_t JMediaSync::setPlaybackParams(const AudioPlaybackRate& rate) {
     82     return mSync->setPlaybackSettings(rate);
     83 }
     84 
     85 void JMediaSync::getPlaybackParams(AudioPlaybackRate* rate /* nonnull */) {
     86     mSync->getPlaybackSettings(rate);
     87 }
     88 
     89 status_t JMediaSync::setSyncParams(const AVSyncSettings& syncParams) {
     90     return mSync->setSyncSettings(syncParams);
     91 }
     92 
     93 void JMediaSync::getSyncParams(AVSyncSettings* syncParams /* nonnull */) {
     94     mSync->getSyncSettings(syncParams);
     95 }
     96 
     97 status_t JMediaSync::setVideoFrameRateHint(float rate) {
     98     return mSync->setVideoFrameRateHint(rate);
     99 }
    100 
    101 float JMediaSync::getVideoFrameRate() {
    102     return mSync->getVideoFrameRate();
    103 }
    104 
    105 void JMediaSync::flush() {
    106     mSync->flush();
    107 }
    108 
    109 status_t JMediaSync::updateQueuedAudioData(
    110         int sizeInBytes, int64_t presentationTimeUs) {
    111     return mSync->updateQueuedAudioData(sizeInBytes, presentationTimeUs);
    112 }
    113 
    114 status_t JMediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
    115     return mSync->getPlayTimeForPendingAudioFrames(outTimeUs);
    116 }
    117 
    118 }  // namespace android
    119 
    120 ////////////////////////////////////////////////////////////////////////////////
    121 
    122 using namespace android;
    123 
    124 static sp<JMediaSync> setMediaSync(JNIEnv *env, jobject thiz, const sp<JMediaSync> &sync) {
    125     sp<JMediaSync> old = (JMediaSync *)env->GetLongField(thiz, gFields.context);
    126     if (sync != NULL) {
    127         sync->incStrong(thiz);
    128     }
    129     if (old != NULL) {
    130         old->decStrong(thiz);
    131     }
    132 
    133     env->SetLongField(thiz, gFields.context, (jlong)sync.get());
    134 
    135     return old;
    136 }
    137 
    138 static sp<JMediaSync> getMediaSync(JNIEnv *env, jobject thiz) {
    139     return (JMediaSync *)env->GetLongField(thiz, gFields.context);
    140 }
    141 
    142 static void android_media_MediaSync_release(JNIEnv *env, jobject thiz) {
    143     setMediaSync(env, thiz, NULL);
    144 }
    145 
    146 static void throwExceptionAsNecessary(
    147         JNIEnv *env, status_t err, const char *msg = NULL) {
    148     switch (err) {
    149         case NO_ERROR:
    150             break;
    151 
    152         case BAD_VALUE:
    153             jniThrowException(env, "java/lang/IllegalArgumentException", msg);
    154             break;
    155 
    156         case NO_INIT:
    157         case INVALID_OPERATION:
    158         default:
    159             if (err > 0) {
    160                 break;
    161             }
    162             AString msgWithErrorCode(msg == NULL ? "" : msg);
    163             msgWithErrorCode.append(" error:");
    164             msgWithErrorCode.append(err);
    165             jniThrowException(env, "java/lang/IllegalStateException", msgWithErrorCode.c_str());
    166             break;
    167     }
    168 }
    169 
    170 static void android_media_MediaSync_native_setSurface(
    171         JNIEnv *env, jobject thiz, jobject jsurface) {
    172     ALOGV("android_media_MediaSync_setSurface");
    173 
    174     sp<JMediaSync> sync = getMediaSync(env, thiz);
    175     if (sync == NULL) {
    176         throwExceptionAsNecessary(env, INVALID_OPERATION);
    177         return;
    178     }
    179 
    180     sp<IGraphicBufferProducer> bufferProducer;
    181     if (jsurface != NULL) {
    182         sp<Surface> surface(android_view_Surface_getSurface(env, jsurface));
    183         if (surface != NULL) {
    184             bufferProducer = surface->getIGraphicBufferProducer();
    185         } else {
    186             throwExceptionAsNecessary(env, BAD_VALUE, "The surface has been released");
    187             return;
    188         }
    189     }
    190 
    191     status_t err = sync->setSurface(bufferProducer);
    192 
    193     if (err == INVALID_OPERATION) {
    194         throwExceptionAsNecessary(
    195                 env, INVALID_OPERATION, "Surface has already been configured");
    196     } if (err != NO_ERROR) {
    197         AString msg("Failed to connect to surface with error ");
    198         msg.append(err);
    199         throwExceptionAsNecessary(env, BAD_VALUE, msg.c_str());
    200     }
    201 }
    202 
    203 static void android_media_MediaSync_native_setAudioTrack(
    204         JNIEnv *env, jobject thiz, jobject jaudioTrack) {
    205     ALOGV("android_media_MediaSync_setAudioTrack");
    206 
    207     sp<JMediaSync> sync = getMediaSync(env, thiz);
    208     if (sync == NULL) {
    209         throwExceptionAsNecessary(env, INVALID_OPERATION);
    210         return;
    211     }
    212 
    213     sp<AudioTrack> audioTrack;
    214     if (jaudioTrack != NULL) {
    215         audioTrack = android_media_AudioTrack_getAudioTrack(env, jaudioTrack);
    216         if (audioTrack == NULL) {
    217             throwExceptionAsNecessary(env, BAD_VALUE, "The audio track has been released");
    218             return;
    219         }
    220     }
    221 
    222     status_t err = sync->setAudioTrack(audioTrack);
    223 
    224     if (err == INVALID_OPERATION) {
    225         throwExceptionAsNecessary(
    226                 env, INVALID_OPERATION, "Audio track has already been configured");
    227     } if (err != NO_ERROR) {
    228         AString msg("Failed to configure audio track with error ");
    229         msg.append(err);
    230         throwExceptionAsNecessary(env, BAD_VALUE, msg.c_str());
    231     }
    232 }
    233 
    234 static jobject android_media_MediaSync_createInputSurface(
    235         JNIEnv* env, jobject thiz) {
    236     ALOGV("android_media_MediaSync_createInputSurface");
    237 
    238     sp<JMediaSync> sync = getMediaSync(env, thiz);
    239     if (sync == NULL) {
    240         throwExceptionAsNecessary(env, INVALID_OPERATION);
    241         return NULL;
    242     }
    243 
    244     // Tell the MediaSync that we want to use a Surface as input.
    245     sp<IGraphicBufferProducer> bufferProducer;
    246     status_t err = sync->createInputSurface(&bufferProducer);
    247     if (err != NO_ERROR) {
    248         throwExceptionAsNecessary(env, INVALID_OPERATION);
    249         return NULL;
    250     }
    251 
    252     // Wrap the IGBP in a Java-language Surface.
    253     return android_view_Surface_createFromIGraphicBufferProducer(env,
    254             bufferProducer);
    255 }
    256 
    257 static void android_media_MediaSync_native_updateQueuedAudioData(
    258         JNIEnv *env, jobject thiz, jint sizeInBytes, jlong presentationTimeUs) {
    259     sp<JMediaSync> sync = getMediaSync(env, thiz);
    260     if (sync == NULL) {
    261         throwExceptionAsNecessary(env, INVALID_OPERATION);
    262         return;
    263     }
    264 
    265     status_t err = sync->updateQueuedAudioData(sizeInBytes, presentationTimeUs);
    266     if (err != NO_ERROR) {
    267         throwExceptionAsNecessary(env, err);
    268         return;
    269     }
    270 }
    271 
    272 static jboolean android_media_MediaSync_native_getTimestamp(
    273         JNIEnv *env, jobject thiz, jobject timestamp) {
    274     sp<JMediaSync> sync = getMediaSync(env, thiz);
    275     if (sync == NULL) {
    276         throwExceptionAsNecessary(env, INVALID_OPERATION);
    277         return JNI_FALSE;
    278     }
    279 
    280     sp<const MediaClock> mediaClock = sync->getMediaClock();
    281     if (mediaClock == NULL) {
    282         return JNI_FALSE;
    283     }
    284 
    285     int64_t nowUs = ALooper::GetNowUs();
    286     int64_t mediaUs = 0;
    287     if (mediaClock->getMediaTime(nowUs, &mediaUs) != OK) {
    288         return JNI_FALSE;
    289     }
    290 
    291     env->SetLongField(timestamp, gFields.mediaTimestampMediaTimeUsID,
    292             (jlong)mediaUs);
    293     env->SetLongField(timestamp, gFields.mediaTimestampNanoTimeID,
    294             (jlong)(nowUs * 1000));
    295     env->SetFloatField(timestamp, gFields.mediaTimestampClockRateID,
    296             (jfloat)mediaClock->getPlaybackRate());
    297     return JNI_TRUE;
    298 }
    299 
    300 static jlong android_media_MediaSync_native_getPlayTimeForPendingAudioFrames(
    301         JNIEnv *env, jobject thiz) {
    302     sp<JMediaSync> sync = getMediaSync(env, thiz);
    303     if (sync == NULL) {
    304         throwExceptionAsNecessary(env, INVALID_OPERATION);
    305     }
    306 
    307     int64_t playTimeUs = 0;
    308     status_t err = sync->getPlayTimeForPendingAudioFrames(&playTimeUs);
    309     if (err != NO_ERROR) {
    310         throwExceptionAsNecessary(env, err);
    311     }
    312     return (jlong)playTimeUs;
    313 }
    314 
    315 static jfloat android_media_MediaSync_setPlaybackParams(
    316         JNIEnv *env, jobject thiz, jobject params) {
    317     sp<JMediaSync> sync = getMediaSync(env, thiz);
    318     if (sync == NULL) {
    319         throwExceptionAsNecessary(env, INVALID_OPERATION);
    320         return (jfloat)0.f;
    321     }
    322 
    323     PlaybackParams pbs;
    324     pbs.fillFromJobject(env, gPlaybackParamsFields, params);
    325     ALOGV("setPlaybackParams: %d:%f %d:%f %d:%u %d:%u",
    326             pbs.speedSet, pbs.audioRate.mSpeed,
    327             pbs.pitchSet, pbs.audioRate.mPitch,
    328             pbs.audioFallbackModeSet, pbs.audioRate.mFallbackMode,
    329             pbs.audioStretchModeSet, pbs.audioRate.mStretchMode);
    330 
    331     AudioPlaybackRate rate;
    332     sync->getPlaybackParams(&rate);
    333     bool updatedRate = false;
    334     if (pbs.speedSet) {
    335         rate.mSpeed = pbs.audioRate.mSpeed;
    336         updatedRate = true;
    337     }
    338     if (pbs.pitchSet) {
    339         rate.mPitch = pbs.audioRate.mPitch;
    340         updatedRate = true;
    341     }
    342     if (pbs.audioFallbackModeSet) {
    343         rate.mFallbackMode = pbs.audioRate.mFallbackMode;
    344         updatedRate = true;
    345     }
    346     if (pbs.audioStretchModeSet) {
    347         rate.mStretchMode = pbs.audioRate.mStretchMode;
    348         updatedRate = true;
    349     }
    350     if (updatedRate) {
    351         status_t err = sync->setPlaybackParams(rate);
    352         if (err != OK) {
    353             throwExceptionAsNecessary(env, err);
    354             return (jfloat)0.f;
    355         }
    356     }
    357 
    358     sp<const MediaClock> mediaClock = sync->getMediaClock();
    359     if (mediaClock == NULL) {
    360         return (jfloat)0.f;
    361     }
    362 
    363     return (jfloat)mediaClock->getPlaybackRate();
    364 }
    365 
    366 static jobject android_media_MediaSync_getPlaybackParams(
    367         JNIEnv *env, jobject thiz) {
    368     sp<JMediaSync> sync = getMediaSync(env, thiz);
    369     if (sync == NULL) {
    370         throwExceptionAsNecessary(env, INVALID_OPERATION);
    371         return NULL;
    372     }
    373 
    374     PlaybackParams pbs;
    375     AudioPlaybackRate &audioRate = pbs.audioRate;
    376     sync->getPlaybackParams(&audioRate);
    377     ALOGV("getPlaybackParams: %f %f %d %d",
    378             audioRate.mSpeed, audioRate.mPitch, audioRate.mFallbackMode, audioRate.mStretchMode);
    379 
    380     pbs.speedSet = true;
    381     pbs.pitchSet = true;
    382     pbs.audioFallbackModeSet = true;
    383     pbs.audioStretchModeSet = true;
    384 
    385     return pbs.asJobject(env, gPlaybackParamsFields);
    386 }
    387 
    388 static jfloat android_media_MediaSync_setSyncParams(
    389         JNIEnv *env, jobject thiz, jobject params) {
    390     sp<JMediaSync> sync = getMediaSync(env, thiz);
    391     if (sync == NULL) {
    392         throwExceptionAsNecessary(env, INVALID_OPERATION);
    393         return (jfloat)0.f;
    394     }
    395 
    396     SyncParams scs;
    397     scs.fillFromJobject(env, gSyncParamsFields, params);
    398     ALOGV("setSyncParams: %d:%d %d:%d %d:%f %d:%f",
    399             scs.syncSourceSet, scs.sync.mSource,
    400             scs.audioAdjustModeSet, scs.sync.mAudioAdjustMode,
    401             scs.toleranceSet, scs.sync.mTolerance,
    402             scs.frameRateSet, scs.frameRate);
    403 
    404     AVSyncSettings avsync;
    405     sync->getSyncParams(&avsync);
    406     bool updatedSync = false;
    407     status_t err = OK;
    408     if (scs.syncSourceSet) {
    409         avsync.mSource = scs.sync.mSource;
    410         updatedSync = true;
    411     }
    412     if (scs.audioAdjustModeSet) {
    413         avsync.mAudioAdjustMode = scs.sync.mAudioAdjustMode;
    414         updatedSync = true;
    415     }
    416     if (scs.toleranceSet) {
    417         avsync.mTolerance = scs.sync.mTolerance;
    418         updatedSync = true;
    419     }
    420     if (updatedSync) {
    421         err = sync->setSyncParams(avsync);
    422     }
    423 
    424     if (scs.frameRateSet && err == OK) {
    425         err = sync->setVideoFrameRateHint(scs.frameRate);
    426     }
    427     if (err != OK) {
    428         throwExceptionAsNecessary(env, err);
    429         return (jfloat)0.f;
    430     }
    431 
    432     sp<const MediaClock> mediaClock = sync->getMediaClock();
    433     if (mediaClock == NULL) {
    434         return (jfloat)0.f;
    435     }
    436 
    437     return (jfloat)mediaClock->getPlaybackRate();
    438 }
    439 
    440 static jobject android_media_MediaSync_getSyncParams(JNIEnv *env, jobject thiz) {
    441     sp<JMediaSync> sync = getMediaSync(env, thiz);
    442     if (sync == NULL) {
    443         throwExceptionAsNecessary(env, INVALID_OPERATION);
    444         return NULL;
    445     }
    446 
    447     SyncParams scs;
    448     sync->getSyncParams(&scs.sync);
    449     scs.frameRate = sync->getVideoFrameRate();
    450 
    451     ALOGV("getSyncParams: %d %d %f %f",
    452             scs.sync.mSource, scs.sync.mAudioAdjustMode, scs.sync.mTolerance, scs.frameRate);
    453 
    454     // sanity check params
    455     if (scs.sync.mSource >= AVSYNC_SOURCE_MAX
    456             || scs.sync.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
    457             || scs.sync.mTolerance < 0.f
    458             || scs.sync.mTolerance >= AVSYNC_TOLERANCE_MAX) {
    459         throwExceptionAsNecessary(env, INVALID_OPERATION);
    460         return NULL;
    461     }
    462 
    463     scs.syncSourceSet = true;
    464     scs.audioAdjustModeSet = true;
    465     scs.toleranceSet = true;
    466     scs.frameRateSet = scs.frameRate >= 0.f;
    467 
    468     return scs.asJobject(env, gSyncParamsFields);
    469 }
    470 
    471 static void android_media_MediaSync_native_flush(JNIEnv *env, jobject thiz) {
    472     sp<JMediaSync> sync = getMediaSync(env, thiz);
    473     if (sync == NULL) {
    474         throwExceptionAsNecessary(env, INVALID_OPERATION);
    475         return;
    476     }
    477 
    478     sync->flush();
    479 }
    480 
    481 static void android_media_MediaSync_native_init(JNIEnv *env) {
    482     ScopedLocalRef<jclass> clazz(env, env->FindClass("android/media/MediaSync"));
    483     CHECK(clazz.get() != NULL);
    484 
    485     gFields.context = env->GetFieldID(clazz.get(), "mNativeContext", "J");
    486     CHECK(gFields.context != NULL);
    487 
    488     clazz.reset(env->FindClass("android/media/MediaTimestamp"));
    489     CHECK(clazz.get() != NULL);
    490 
    491     gFields.mediaTimestampMediaTimeUsID =
    492         env->GetFieldID(clazz.get(), "mediaTimeUs", "J");
    493     CHECK(gFields.mediaTimestampMediaTimeUsID != NULL);
    494 
    495     gFields.mediaTimestampNanoTimeID =
    496         env->GetFieldID(clazz.get(), "nanoTime", "J");
    497     CHECK(gFields.mediaTimestampNanoTimeID != NULL);
    498 
    499     gFields.mediaTimestampClockRateID =
    500         env->GetFieldID(clazz.get(), "clockRate", "F");
    501     CHECK(gFields.mediaTimestampClockRateID != NULL);
    502 
    503     gSyncParamsFields.init(env);
    504     gPlaybackParamsFields.init(env);
    505 }
    506 
    507 static void android_media_MediaSync_native_setup(JNIEnv *env, jobject thiz) {
    508     sp<JMediaSync> sync = new JMediaSync();
    509 
    510     setMediaSync(env, thiz, sync);
    511 }
    512 
    513 static void android_media_MediaSync_native_finalize(JNIEnv *env, jobject thiz) {
    514     android_media_MediaSync_release(env, thiz);
    515 }
    516 
    517 static JNINativeMethod gMethods[] = {
    518     { "native_setSurface",
    519       "(Landroid/view/Surface;)V",
    520       (void *)android_media_MediaSync_native_setSurface },
    521 
    522     { "native_setAudioTrack",
    523       "(Landroid/media/AudioTrack;)V",
    524       (void *)android_media_MediaSync_native_setAudioTrack },
    525 
    526     { "createInputSurface", "()Landroid/view/Surface;",
    527       (void *)android_media_MediaSync_createInputSurface },
    528 
    529     { "native_updateQueuedAudioData",
    530       "(IJ)V",
    531       (void *)android_media_MediaSync_native_updateQueuedAudioData },
    532 
    533     { "native_getTimestamp",
    534       "(Landroid/media/MediaTimestamp;)Z",
    535       (void *)android_media_MediaSync_native_getTimestamp },
    536 
    537     { "native_getPlayTimeForPendingAudioFrames",
    538       "()J",
    539       (void *)android_media_MediaSync_native_getPlayTimeForPendingAudioFrames },
    540 
    541     { "native_flush", "()V", (void *)android_media_MediaSync_native_flush },
    542 
    543     { "native_init", "()V", (void *)android_media_MediaSync_native_init },
    544 
    545     { "native_setup", "()V", (void *)android_media_MediaSync_native_setup },
    546 
    547     { "native_release", "()V", (void *)android_media_MediaSync_release },
    548 
    549     { "native_setPlaybackParams", "(Landroid/media/PlaybackParams;)F",
    550       (void *)android_media_MediaSync_setPlaybackParams },
    551 
    552     { "getPlaybackParams", "()Landroid/media/PlaybackParams;",
    553       (void *)android_media_MediaSync_getPlaybackParams },
    554 
    555     { "native_setSyncParams", "(Landroid/media/SyncParams;)F",
    556       (void *)android_media_MediaSync_setSyncParams },
    557 
    558     { "getSyncParams", "()Landroid/media/SyncParams;",
    559       (void *)android_media_MediaSync_getSyncParams },
    560 
    561     { "native_finalize", "()V", (void *)android_media_MediaSync_native_finalize },
    562 };
    563 
    564 int register_android_media_MediaSync(JNIEnv *env) {
    565     return AndroidRuntime::registerNativeMethods(
    566                    env, "android/media/MediaSync", gMethods, NELEM(gMethods));
    567 }
    568