Home | History | Annotate | Download | only in loopback 
      1 /*
      2  * Copyright (C) 2015 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package org.drrickorang.loopback;
     18 
     19 import java.nio.ByteBuffer;
     20 import java.util.Arrays;
     21 
     22 import android.util.Log;
     23 import android.os.Handler;
     24 import android.os.Message;
     25 
     26 
     27 /**
     28  * A thread/audio track based audio synth.
     29  */
     30 
     31 public class NativeAudioThread extends Thread {
     32     private static final String TAG = "NativeAudioThread";
     33 
     34     // for latency test
     35     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_STARTED = 891;
     36     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_ERROR = 892;
     37     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_COMPLETE = 893;
     38     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_COMPLETE_ERRORS = 894;
     39     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_STOP = 895;
     40 
     41     // for buffer test
     42     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_STARTED = 896;
     43     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_ERROR = 897;
     44     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_COMPLETE = 898;
     45     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_COMPLETE_ERRORS = 899;
     46     static final int LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_STOP = 900;
     47 
     48     public boolean  mIsRunning = false;
     49     public int      mSessionId;
     50     public double[] mSamples; // store samples that will be shown on WavePlotView
     51     int             mSamplesIndex;
     52 
     53     private int mTestType;
     54     private int mSamplingRate;
     55     private int mMinPlayerBufferSizeInBytes = 0;
     56     private int mMinRecorderBuffSizeInBytes = 0; // currently not used
     57     private int mMicSource;
     58     private int mPerformanceMode = -1;
     59     private int mIgnoreFirstFrames;
     60 
     61     private boolean mIsRequestStop = false;
     62     private Handler mMessageHandler;
     63     private boolean isDestroying = false;
     64     private boolean hasDestroyingErrors = false;
     65 
     66     // for buffer test
     67     private int[]   mRecorderBufferPeriod;
     68     private int     mRecorderMaxBufferPeriod;
     69     private double  mRecorderStdDevBufferPeriod;
     70     private int[]   mPlayerBufferPeriod;
     71     private int     mPlayerMaxBufferPeriod;
     72     private double  mPlayerStdDevBufferPeriod;
     73     private BufferCallbackTimes mPlayerCallbackTimes;
     74     private BufferCallbackTimes mRecorderCallbackTimes;
     75     private int     mBufferTestWavePlotDurationInSeconds;
     76     private double  mFrequency1 = Constant.PRIME_FREQUENCY_1;
     77     private double  mFrequency2 = Constant.PRIME_FREQUENCY_2; // not actually used
     78     private int     mBufferTestDurationInSeconds;
     79     private int     mFFTSamplingSize;
     80     private int     mFFTOverlapSamples;
     81     private int[]   mAllGlitches;
     82     private boolean mGlitchingIntervalTooLong;
     83     private final CaptureHolder mCaptureHolder;
     84 
     85     private PipeByteBuffer        mPipeByteBuffer;
     86     private GlitchDetectionThread mGlitchDetectionThread;
     87 
     88 
     89     public NativeAudioThread(int samplingRate, int playerBufferInBytes, int recorderBufferInBytes,
     90                              int micSource, int performanceMode, int testType, int bufferTestDurationInSeconds,
     91                              int bufferTestWavePlotDurationInSeconds, int ignoreFirstFrames,
     92                              CaptureHolder captureHolder) {
     93         mSamplingRate = samplingRate;
     94         mMinPlayerBufferSizeInBytes = playerBufferInBytes;
     95         mMinRecorderBuffSizeInBytes = recorderBufferInBytes;
     96         mMicSource = micSource;
     97         mPerformanceMode = performanceMode;
     98         mTestType = testType;
     99         mBufferTestDurationInSeconds = bufferTestDurationInSeconds;
    100         mBufferTestWavePlotDurationInSeconds = bufferTestWavePlotDurationInSeconds;
    101         mIgnoreFirstFrames = ignoreFirstFrames;
    102         mCaptureHolder = captureHolder;
    103         setName("Loopback_NativeAudio");
    104     }
    105 
    106     public NativeAudioThread(NativeAudioThread old) {
    107         mSamplingRate = old.mSamplingRate;
    108         mMinPlayerBufferSizeInBytes = old.mMinPlayerBufferSizeInBytes;
    109         mMinRecorderBuffSizeInBytes = old.mMinRecorderBuffSizeInBytes;
    110         mMicSource = old.mMicSource;
    111         mPerformanceMode = old.mPerformanceMode;
    112         mTestType = old.mTestType;
    113         mBufferTestDurationInSeconds = old.mBufferTestDurationInSeconds;
    114         mBufferTestWavePlotDurationInSeconds = old.mBufferTestWavePlotDurationInSeconds;
    115         mIgnoreFirstFrames = old.mIgnoreFirstFrames;
    116         mCaptureHolder = old.mCaptureHolder;
    117         setName("Loopback_NativeAudio");
    118     }
    119 
    120     //JNI load
    121     static {
    122         try {
    123             System.loadLibrary("loopback");
    124         } catch (UnsatisfiedLinkError e) {
    125             log("Error loading loopback JNI library");
    126             e.printStackTrace();
    127         }
    128         /* TODO: gracefully fail/notify if the library can't be loaded */
    129     }
    130 
    131 
    132     //jni calls
    133     public native long  slesInit(int samplingRate, int frameCount, int micSource,
    134                                  int performanceMode,
    135                                  int testType, double frequency1, ByteBuffer byteBuffer,
    136                                  short[] sincTone, int maxRecordedLateCallbacks,
    137                                  int ignoreFirstFrames);
    138     public native int   slesProcessNext(long sles_data, double[] samples, long offset);
    139     public native int   slesDestroy(long sles_data);
    140 
    141     // to get buffer period data
    142     public native int[]  slesGetRecorderBufferPeriod(long sles_data);
    143     public native int    slesGetRecorderMaxBufferPeriod(long sles_data);
    144     public native double slesGetRecorderVarianceBufferPeriod(long sles_data);
    145     public native int[]  slesGetPlayerBufferPeriod(long sles_data);
    146     public native int    slesGetPlayerMaxBufferPeriod(long sles_data);
    147     public native double slesGetPlayerVarianceBufferPeriod(long sles_data);
    148     public native BufferCallbackTimes slesGetPlayerCallbackTimeStamps(long sles_data);
    149     public native BufferCallbackTimes slesGetRecorderCallbackTimeStamps(long sles_data);
    150 
    151     public native int slesGetCaptureRank(long sles_data);
    152 
    153 
    154     public void run() {
    155         setPriority(Thread.MAX_PRIORITY);
    156         mIsRunning = true;
    157 
    158         //erase output buffer
    159         if (mSamples != null)
    160             mSamples = null;
    161 
    162         //start playing
    163         log(" Started capture test");
    164         if (mMessageHandler != null) {
    165             Message msg = Message.obtain();
    166             switch (mTestType) {
    167             case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY:
    168                 msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_STARTED;
    169                 break;
    170             case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_BUFFER_PERIOD:
    171                 msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_STARTED;
    172                 break;
    173             }
    174             mMessageHandler.sendMessage(msg);
    175         }
    176 
    177         //generate sinc tone use for loopback test
    178         short loopbackTone[] = new short[mMinPlayerBufferSizeInBytes / Constant.BYTES_PER_FRAME];
    179         if (mTestType == Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY) {
    180             ToneGeneration sincToneGen = new RampedSineTone(mSamplingRate,
    181                     Constant.LOOPBACK_FREQUENCY);
    182             sincToneGen.generateTone(loopbackTone, loopbackTone.length);
    183         }
    184 
    185         log(String.format("about to init, sampling rate: %d, buffer:%d", mSamplingRate,
    186                 mMinPlayerBufferSizeInBytes / Constant.BYTES_PER_FRAME));
    187 
    188         // mPipeByteBuffer is only used in buffer test
    189         mPipeByteBuffer = new PipeByteBuffer(Constant.MAX_SHORTS);
    190         long startTimeMs = System.currentTimeMillis();
    191         long sles_data = slesInit(mSamplingRate,
    192                 mMinPlayerBufferSizeInBytes / Constant.BYTES_PER_FRAME, mMicSource,
    193                 mPerformanceMode, mTestType,
    194                 mFrequency1, mPipeByteBuffer.getByteBuffer(), loopbackTone,
    195                 mBufferTestDurationInSeconds * Constant.MAX_RECORDED_LATE_CALLBACKS_PER_SECOND,
    196                 mIgnoreFirstFrames);
    197         log(String.format("sles_data = 0x%X", sles_data));
    198 
    199         if (sles_data == 0) {
    200             //notify error!!
    201             log(" ERROR at JNI initialization");
    202             if (mMessageHandler != null) {
    203                 Message msg = Message.obtain();
    204                 switch (mTestType) {
    205                 case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY:
    206                     msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_ERROR;
    207                     break;
    208                 case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_BUFFER_PERIOD:
    209                     msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_ERROR;
    210                     break;
    211                 }
    212                 mMessageHandler.sendMessage(msg);
    213             }
    214         } else {
    215             // wait a little bit
    216             try {
    217                 final int setUpTime = 10;
    218                 sleep(setUpTime); //just to let it start properly
    219             } catch (InterruptedException e) {
    220                 e.printStackTrace();
    221             }
    222 
    223 
    224             int totalSamplesRead = 0;
    225             switch (mTestType) {
    226             case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY:
    227                 final int latencyTestDurationInSeconds = 2;
    228                 int nNewSize = (int) (1.1 * mSamplingRate * latencyTestDurationInSeconds);
    229                 mSamples = new double[nNewSize];
    230                 mSamplesIndex = 0; //reset index
    231                 Arrays.fill(mSamples, 0);
    232 
    233                 //TODO use a ByteBuffer to retrieve recorded data instead
    234                 long offset = 0;
    235                 // retrieve native recorder's recorded data
    236                 for (int ii = 0; ii < latencyTestDurationInSeconds; ii++) {
    237                     log(String.format("block %d...", ii));
    238                     int samplesRead = slesProcessNext(sles_data, mSamples, offset);
    239                     totalSamplesRead += samplesRead;
    240                     offset += samplesRead;
    241                     log(" [" + ii + "] jni samples read:" + samplesRead +
    242                         "  currentOffset:" + offset);
    243                 }
    244 
    245                 log(String.format(" samplesRead: %d, sampleOffset:%d", totalSamplesRead, offset));
    246                 log("about to destroy...");
    247                 break;
    248             case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_BUFFER_PERIOD:
    249                 setUpGlitchDetectionThread();
    250                 long testDurationMs = mBufferTestDurationInSeconds * Constant.MILLIS_PER_SECOND;
    251                 long elapsedTimeMs = System.currentTimeMillis() - startTimeMs;
    252                 while (elapsedTimeMs < testDurationMs) {
    253                     if (mIsRequestStop) {
    254                         break;
    255                     } else {
    256                         int rank = slesGetCaptureRank(sles_data);
    257                         if (rank > 0) {
    258                             //log("Late callback detected");
    259                             mCaptureHolder.captureState(rank);
    260                         }
    261                         try {
    262                             final int setUpTime = 100;
    263                             sleep(setUpTime); //just to let it start properly
    264                         } catch (InterruptedException e) {
    265                             e.printStackTrace();
    266                         }
    267                         elapsedTimeMs = System.currentTimeMillis() - startTimeMs;
    268                     }
    269 
    270                 }
    271                 break;
    272 
    273 
    274             }
    275 
    276             // collect buffer period data
    277             mRecorderBufferPeriod = slesGetRecorderBufferPeriod(sles_data);
    278             mRecorderMaxBufferPeriod = slesGetRecorderMaxBufferPeriod(sles_data);
    279             mRecorderStdDevBufferPeriod = Math.sqrt(slesGetRecorderVarianceBufferPeriod(sles_data));
    280             mPlayerBufferPeriod = slesGetPlayerBufferPeriod(sles_data);
    281             mPlayerMaxBufferPeriod = slesGetPlayerMaxBufferPeriod(sles_data);
    282             mPlayerStdDevBufferPeriod = Math.sqrt(slesGetPlayerVarianceBufferPeriod(sles_data));
    283 
    284             mPlayerCallbackTimes = slesGetPlayerCallbackTimeStamps(sles_data);
    285             mRecorderCallbackTimes = slesGetRecorderCallbackTimeStamps(sles_data);
    286 
    287             // get glitches data only for buffer test
    288             if (mTestType == Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_BUFFER_PERIOD) {
    289                 mAllGlitches = mGlitchDetectionThread.getGlitches();
    290                 mSamples = mGlitchDetectionThread.getWaveData();
    291                 mGlitchingIntervalTooLong = mGlitchDetectionThread.getGlitchingIntervalTooLong();
    292                 endDetecting();
    293             }
    294 
    295             if (mTestType == Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY) {
    296                 mCaptureHolder.captureState(0);
    297             }
    298 
    299             runDestroy(sles_data);
    300 
    301             final int maxTry = 20;
    302             int tryCount = 0;
    303             while (isDestroying) {
    304                 try {
    305                     sleep(40);
    306                 } catch (InterruptedException e) {
    307                     e.printStackTrace();
    308                 }
    309 
    310                 tryCount++;
    311                 log("destroy try: " + tryCount);
    312 
    313                 if (tryCount >= maxTry) {
    314                     hasDestroyingErrors = true;
    315                     log("WARNING: waited for max time to properly destroy JNI.");
    316                     break;
    317                 }
    318             }
    319             log(String.format("after destroying. TotalSamplesRead = %d", totalSamplesRead));
    320 
    321             // for buffer test samples won't be read into here
    322             if (mTestType == Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY
    323                 && totalSamplesRead == 0) {
    324                 //hasDestroyingErrors = true;
    325                 log("Warning: Latency test reads no sample from native recorder!");
    326             }
    327 
    328             endTest();
    329         }
    330     }
    331 
    332 
    333     public void requestStopTest() {
    334         mIsRequestStop = true;
    335     }
    336 
    337 
    338     /** Set up parameters needed for GlitchDetectionThread, then create and run this thread. */
    339     private void setUpGlitchDetectionThread() {
    340         final int targetFFTMs = 20; // we want each FFT to cover 20ms of samples
    341         mFFTSamplingSize = targetFFTMs * mSamplingRate / Constant.MILLIS_PER_SECOND;
    342         // round to the nearest power of 2
    343         mFFTSamplingSize = (int) Math.pow(2, Math.round(Math.log(mFFTSamplingSize) / Math.log(2)));
    344 
    345         if (mFFTSamplingSize < 2) {
    346             mFFTSamplingSize = 2; // mFFTSamplingSize should be at least 2
    347         }
    348         mFFTOverlapSamples = mFFTSamplingSize / 2; // mFFTOverlapSamples is half of mFFTSamplingSize
    349 
    350         mGlitchDetectionThread = new GlitchDetectionThread(mFrequency1, mFrequency2, mSamplingRate,
    351             mFFTSamplingSize, mFFTOverlapSamples, mBufferTestDurationInSeconds,
    352             mBufferTestWavePlotDurationInSeconds, mPipeByteBuffer, mCaptureHolder);
    353         mGlitchDetectionThread.start();
    354     }
    355 
    356 
    357     public void endDetecting() {
    358         mPipeByteBuffer.flush();
    359         mPipeByteBuffer = null;
    360         mGlitchDetectionThread.requestStop();
    361         GlitchDetectionThread tempThread = mGlitchDetectionThread;
    362         mGlitchDetectionThread = null;
    363         try {
    364             tempThread.join(Constant.JOIN_WAIT_TIME_MS);
    365         } catch (InterruptedException e) {
    366             e.printStackTrace();
    367         }
    368     }
    369 
    370 
    371     public void setMessageHandler(Handler messageHandler) {
    372         mMessageHandler = messageHandler;
    373     }
    374 
    375 
    376     private void runDestroy(final long sles_data) {
    377         isDestroying = true;
    378 
    379         //start thread
    380         final long local_sles_data = sles_data;
    381         Thread thread = new Thread(new Runnable() {
    382             public void run() {
    383                 isDestroying = true;
    384                 log("**Start runnable destroy");
    385 
    386                 int status = slesDestroy(local_sles_data);
    387                 log(String.format("**End runnable destroy sles delete status: %d", status));
    388                 isDestroying = false;
    389             }
    390         });
    391 
    392         thread.start();
    393         log("end of runDestroy()");
    394     }
    395 
    396 
    397     /** not doing real work, just to keep consistency with LoopbackAudioThread. */
    398     public void runTest() {
    399 
    400     }
    401 
    402 
    403     /** not doing real work, just to keep consistency with LoopbackAudioThread. */
    404     public void runBufferTest() {
    405 
    406     }
    407 
    408 
    409     public void endTest() {
    410        log("--Ending capture test--");
    411        if (mMessageHandler != null) {
    412            Message msg = Message.obtain();
    413            if (hasDestroyingErrors) {
    414                switch (mTestType) {
    415                    case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY:
    416                        msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_COMPLETE_ERRORS;
    417                        break;
    418                    case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_BUFFER_PERIOD:
    419                        msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_COMPLETE_ERRORS;
    420                        break;
    421                }
    422            } else if (mIsRequestStop) {
    423                switch (mTestType) {
    424                    case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY:
    425                        msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_STOP;
    426                        break;
    427                    case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_BUFFER_PERIOD:
    428                        msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_STOP;
    429                        break;
    430                }
    431            } else {
    432                switch (mTestType) {
    433                case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_LATENCY:
    434                    msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_LATENCY_REC_COMPLETE;
    435                    break;
    436                case Constant.LOOPBACK_PLUG_AUDIO_THREAD_TEST_TYPE_BUFFER_PERIOD:
    437                    msg.what = LOOPBACK_NATIVE_AUDIO_THREAD_MESSAGE_BUFFER_REC_COMPLETE;
    438                    break;
    439                }
    440            }
    441 
    442            mMessageHandler.sendMessage(msg);
    443        }
    444     }
    445 
    446 
    447     public void finish() {
    448         mIsRunning = false;
    449     }
    450 
    451 
    452     private static void log(String msg) {
    453         Log.v(TAG, msg);
    454     }
    455 
    456 
    457     double[] getWaveData() {
    458         return mSamples;
    459     }
    460 
    461 
    462     public int[] getRecorderBufferPeriod() {
    463         return mRecorderBufferPeriod;
    464     }
    465 
    466     public int getRecorderMaxBufferPeriod() {
    467         return mRecorderMaxBufferPeriod;
    468     }
    469 
    470     public double getRecorderStdDevBufferPeriod() {
    471         return mRecorderStdDevBufferPeriod;
    472     }
    473 
    474     public int[] getPlayerBufferPeriod() {
    475         return mPlayerBufferPeriod;
    476     }
    477 
    478     public int getPlayerMaxBufferPeriod() {
    479         return mPlayerMaxBufferPeriod;
    480     }
    481 
    482     public double getPlayerStdDevBufferPeriod() {
    483         return mPlayerStdDevBufferPeriod;
    484     }
    485 
    486     public int[] getNativeAllGlitches() {
    487         return mAllGlitches;
    488     }
    489 
    490 
    491     public boolean getGlitchingIntervalTooLong() {
    492         return mGlitchingIntervalTooLong;
    493     }
    494 
    495 
    496     public int getNativeFFTSamplingSize() {
    497         return mFFTSamplingSize;
    498     }
    499 
    500 
    501     public int getNativeFFTOverlapSamples() {
    502         return mFFTOverlapSamples;
    503     }
    504 
    505 
    506     public int getDurationInSeconds() {
    507         return mBufferTestDurationInSeconds;
    508     }
    509 
    510     public BufferCallbackTimes getPlayerCallbackTimes() {
    511         return mPlayerCallbackTimes;
    512     }
    513 
    514     public BufferCallbackTimes getRecorderCallbackTimes() {
    515         return mRecorderCallbackTimes;
    516     }
    517 }
    518