Home | History | Annotate | Download | only in media
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package android.media;
     18 
     19 import android.annotation.IntDef;
     20 import android.annotation.NonNull;
     21 import android.annotation.Nullable;
     22 import android.graphics.ImageFormat;
     23 import android.graphics.Rect;
     24 import android.graphics.SurfaceTexture;
     25 import android.media.MediaCodecInfo.CodecCapabilities;
     26 import android.os.Bundle;
     27 import android.os.Handler;
     28 import android.os.IBinder;
     29 import android.os.IHwBinder;
     30 import android.os.Looper;
     31 import android.os.Message;
     32 import android.os.PersistableBundle;
     33 import android.view.Surface;
     34 
     35 import java.io.IOException;
     36 import java.lang.annotation.Retention;
     37 import java.lang.annotation.RetentionPolicy;
     38 import java.nio.ByteBuffer;
     39 import java.nio.ByteOrder;
     40 import java.nio.ReadOnlyBufferException;
     41 import java.util.Arrays;
     42 import java.util.HashMap;
     43 import java.util.Map;
     44 
     45 /**
     46  MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components.
     47  It is part of the Android low-level multimedia support infrastructure (normally used together
     48  with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto},
     49  {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.)
     50  <p>
     51  <center><object style="width: 540px; height: 205px;" type="image/svg+xml"
     52    data="../../../images/media/mediacodec_buffers.svg"><img
     53    src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px"
     54    alt="MediaCodec buffer flow diagram"></object></center>
     55  <p>
     56  In broad terms, a codec processes input data to generate output data. It processes data
     57  asynchronously and uses a set of input and output buffers. At a simplistic level, you request
     58  (or receive) an empty input buffer, fill it up with data and send it to the codec for
     59  processing. The codec uses up the data and transforms it into one of its empty output buffers.
     60  Finally, you request (or receive) a filled output buffer, consume its contents and release it
     61  back to the codec.
     62 
     63  <h3>Data Types</h3>
     64  <p>
     65  Codecs operate on three kinds of data: compressed data, raw audio data and raw video data.
     66  All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use
     67  a {@link Surface} for raw video data to improve codec performance. Surface uses native video
     68  buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient.
     69  You normally cannot access the raw video data when using a Surface, but you can use the
     70  {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more
     71  efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain
     72  ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video
     73  frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage
     74  OutputImage(int)}.
     75 
     76  <h4>Compressed Buffers</h4>
     77  <p>
     78  Input buffers (for decoders) and output buffers (for encoders) contain compressed data according
     79  to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single
     80  compressed video frame. For audio data this is normally a single access unit (an encoded audio
     81  segment typically containing a few milliseconds of audio as dictated by the format type), but
     82  this requirement is slightly relaxed in that a buffer may contain multiple encoded access units
     83  of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on
     84  frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}.
     85 
     86  <h4>Raw Audio Buffers</h4>
     87  <p>
     88  Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel
     89  in channel order. Each sample is a {@linkplain AudioFormat#ENCODING_PCM_16BIT 16-bit signed
     90  integer in native byte order}.
     91 
     92  <pre class=prettyprint>
     93  short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) {
     94    ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId);
     95    MediaFormat format = codec.getOutputFormat(bufferId);
     96    ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer();
     97    int numChannels = formet.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
     98    if (channelIx &lt; 0 || channelIx &gt;= numChannels) {
     99      return null;
    100    }
    101    short[] res = new short[samples.remaining() / numChannels];
    102    for (int i = 0; i &lt; res.length; ++i) {
    103      res[i] = samples.get(i * numChannels + channelIx);
    104    }
    105    return res;
    106  }</pre>
    107 
    108  <h4>Raw Video Buffers</h4>
    109  <p>
    110  In ByteBuffer mode video buffers are laid out according to their {@linkplain
    111  MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array
    112  from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType
    113  getCapabilitiesForType(&hellip;)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}.
    114  Video codecs may support three kinds of color formats:
    115  <ul>
    116  <li><strong>native raw video format:</strong> This is marked by {@link
    117  CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li>
    118  <li><strong>flexible YUV buffers</strong> (such as {@link
    119  CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface,
    120  as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage
    121  OutputImage(int)}.</li>
    122  <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer
    123  mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}.
    124  For color formats that are equivalent to a flexible format, you can still use {@link
    125  #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li>
    126  </ul>
    127  <p>
    128  All video codecs support flexible YUV 4:2:0 buffers since {@link
    129  android.os.Build.VERSION_CODES#LOLLIPOP_MR1}.
    130 
    131  <h4>Accessing Raw Video ByteBuffers on Older Devices</h4>
    132  <p>
    133  Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to
    134  use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format
    135  values to understand the layout of the raw output buffers.
    136  <p class=note>
    137  Note that on some devices the slice-height is advertised as 0. This could mean either that the
    138  slice-height is the same as the frame height, or that the slice-height is the frame height
    139  aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way
    140  to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U}
    141  plane in planar formats is also not specified or defined, though usually it is half of the slice
    142  height.
    143  <p>
    144  The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the
    145  video frames; however, for most encondings the video (picture) only occupies a portion of the
    146  video frame. This is represented by the 'crop rectangle'.
    147  <p>
    148  You need to use the following keys to get the crop rectangle of raw output images from the
    149  {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the
    150  entire video frame.The crop rectangle is understood in the context of the output frame
    151  <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}.
    152  <table style="width: 0%">
    153   <thead>
    154    <tr>
    155     <th>Format Key</th>
    156     <th>Type</th>
    157     <th>Description</th>
    158    </tr>
    159   </thead>
    160   <tbody>
    161    <tr>
    162     <td>{@code "crop-left"}</td>
    163     <td>Integer</td>
    164     <td>The left-coordinate (x) of the crop rectangle</td>
    165    </tr><tr>
    166     <td>{@code "crop-top"}</td>
    167     <td>Integer</td>
    168     <td>The top-coordinate (y) of the crop rectangle</td>
    169    </tr><tr>
    170     <td>{@code "crop-right"}</td>
    171     <td>Integer</td>
    172     <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td>
    173    </tr><tr>
    174     <td>{@code "crop-bottom"}</td>
    175     <td>Integer</td>
    176     <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td>
    177    </tr><tr>
    178     <td colspan=3>
    179      The right and bottom coordinates can be understood as the coordinates of the right-most
    180      valid column/bottom-most valid row of the cropped output image.
    181     </td>
    182    </tr>
    183   </tbody>
    184  </table>
    185  <p>
    186  The size of the video frame (before rotation) can be calculated as such:
    187  <pre class=prettyprint>
    188  MediaFormat format = decoder.getOutputFormat(&hellip;);
    189  int width = format.getInteger(MediaFormat.KEY_WIDTH);
    190  if (format.containsKey("crop-left") && format.containsKey("crop-right")) {
    191      width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left");
    192  }
    193  int height = format.getInteger(MediaFormat.KEY_HEIGHT);
    194  if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) {
    195      height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top");
    196  }
    197  </pre>
    198  <p class=note>
    199  Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across
    200  devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on
    201  most devices it pointed to the top-left pixel of the entire frame.
    202 
    203  <h3>States</h3>
    204  <p>
    205  During its life a codec conceptually exists in one of three states: Stopped, Executing or
    206  Released. The Stopped collective state is actually the conglomeration of three states:
    207  Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through
    208  three sub-states: Flushed, Running and End-of-Stream.
    209  <p>
    210  <center><object style="width: 516px; height: 353px;" type="image/svg+xml"
    211    data="../../../images/media/mediacodec_states.svg"><img
    212    src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px"
    213    alt="MediaCodec state diagram"></object></center>
    214  <p>
    215  When you create a codec using one of the factory methods, the codec is in the Uninitialized
    216  state. First, you need to configure it via {@link #configure configure(&hellip;)}, which brings
    217  it to the Configured state, then call {@link #start} to move it to the Executing state. In this
    218  state you can process data through the buffer queue manipulation described above.
    219  <p>
    220  The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after
    221  {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon
    222  as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends
    223  most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM
    224  end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the
    225  codec no longer accepts further input buffers, but still generates output buffers until the
    226  end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time
    227  while in the Executing state using {@link #flush}.
    228  <p>
    229  Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured
    230  again. When you are done using a codec, you must release it by calling {@link #release}.
    231  <p>
    232  On rare occasions the codec may encounter an error and move to the Error state. This is
    233  communicated using an invalid return value from a queuing operation, or sometimes via an
    234  exception. Call {@link #reset} to make the codec usable again. You can call it from any state to
    235  move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the
    236  terminal Released state.
    237 
    238  <h3>Creation</h3>
    239  <p>
    240  Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When
    241  decoding a file or a stream, you can get the desired format from {@link
    242  MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that
    243  you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then
    244  call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the
    245  name of a codec that can handle that specific media format. Finally, create the codec using
    246  {@link #createByCodecName}.
    247  <p class=note>
    248  <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to
    249  {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain
    250  MediaFormat#KEY_FRAME_RATE frame rate}. Use
    251  <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code>
    252  to clear any existing frame rate setting in the format.
    253  <p>
    254  You can also create the preferred codec for a specific MIME type using {@link
    255  #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}.
    256  This, however, cannot be used to inject features, and may create a codec that cannot handle the
    257  specific desired media format.
    258 
    259  <h4>Creating secure decoders</h4>
    260  <p>
    261  On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might
    262  not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs
    263  that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a
    264  regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link
    265  #createByCodecName} will throw an {@code IOException} if the codec is not present on the system.
    266  <p>
    267  From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link
    268  CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder.
    269 
    270  <h3>Initialization</h3>
    271  <p>
    272  After creating the codec, you can set a callback using {@link #setCallback setCallback} if you
    273  want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the
    274  specific media format. This is when you can specify the output {@link Surface} for video
    275  producers &ndash; codecs that generate raw video data (e.g. video decoders). This is also when
    276  you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since
    277  some codecs can operate in multiple modes, you must specify whether you want it to work as a
    278  decoder or an encoder.
    279  <p>
    280  Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and
    281  output format in the Configured state. You can use this to verify the resulting configuration,
    282  e.g. color formats, before starting the codec.
    283  <p>
    284  If you want to process raw input video buffers natively with a video consumer &ndash; a codec
    285  that processes raw video input, such as a video encoder &ndash; create a destination Surface for
    286  your input data using {@link #createInputSurface} after configuration. Alternately, set up the
    287  codec to use a previously created {@linkplain #createPersistentInputSurface persistent input
    288  surface} by calling {@link #setInputSurface}.
    289 
    290  <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4>
    291  <p>
    292  Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data
    293  to be prefixed by a number of buffers containing setup data, or codec specific data. When
    294  processing such compressed formats, this data must be submitted to the codec after {@link
    295  #start} and before any frame data. Such data must be marked using the flag {@link
    296  #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}.
    297  <p>
    298  Codec-specific data can also be included in the format passed to {@link #configure configure} in
    299  ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track
    300  {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}.
    301  Codec-specific data in the format is automatically submitted to the codec upon {@link #start};
    302  you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec
    303  specific data, you can choose to submit it using the specified number of buffers in the correct
    304  order, according to the format requirements. In case of H.264 AVC, you can also concatenate all
    305  codec-specific data and submit it as a single codec-config buffer.
    306  <p>
    307  Android uses the following codec-specific data buffers. These are also required to be set in
    308  the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the
    309  codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of
    310  {@code "\x00\x00\x00\x01"}.
    311  <p>
    312  <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style>
    313  <table>
    314   <thead>
    315    <th>Format</th>
    316    <th>CSD buffer #0</th>
    317    <th>CSD buffer #1</th>
    318    <th>CSD buffer #2</th>
    319   </thead>
    320   <tbody class=mid>
    321    <tr>
    322     <td>AAC</td>
    323     <td>Decoder-specific information from ESDS<sup>*</sup></td>
    324     <td class=NA>Not Used</td>
    325     <td class=NA>Not Used</td>
    326    </tr>
    327    <tr>
    328     <td>VORBIS</td>
    329     <td>Identification header</td>
    330     <td>Setup header</td>
    331     <td class=NA>Not Used</td>
    332    </tr>
    333    <tr>
    334     <td>OPUS</td>
    335     <td>Identification header</td>
    336     <td>Pre-skip in nanosecs<br>
    337         (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br>
    338         This overrides the pre-skip value in the identification header.</td>
    339     <td>Seek Pre-roll in nanosecs<br>
    340         (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td>
    341    </tr>
    342    <tr>
    343     <td>FLAC</td>
    344     <td>mandatory metadata block (called the STREAMINFO block),<br>
    345         optionally followed by any number of other metadata blocks</td>
    346     <td class=NA>Not Used</td>
    347     <td class=NA>Not Used</td>
    348    </tr>
    349    <tr>
    350     <td>MPEG-4</td>
    351     <td>Decoder-specific information from ESDS<sup>*</sup></td>
    352     <td class=NA>Not Used</td>
    353     <td class=NA>Not Used</td>
    354    </tr>
    355    <tr>
    356     <td>H.264 AVC</td>
    357     <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td>
    358     <td>PPS (Picture Parameter Sets<sup>*</sup>)</td>
    359     <td class=NA>Not Used</td>
    360    </tr>
    361    <tr>
    362     <td>H.265 HEVC</td>
    363     <td>VPS (Video Parameter Sets<sup>*</sup>) +<br>
    364      SPS (Sequence Parameter Sets<sup>*</sup>) +<br>
    365      PPS (Picture Parameter Sets<sup>*</sup>)</td>
    366     <td class=NA>Not Used</td>
    367     <td class=NA>Not Used</td>
    368    </tr>
    369    <tr>
    370     <td>VP9</td>
    371     <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data
    372         (optional)</td>
    373     <td class=NA>Not Used</td>
    374     <td class=NA>Not Used</td>
    375    </tr>
    376   </tbody>
    377  </table>
    378 
    379  <p class=note>
    380  <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly
    381  after start, before any output buffer or output format change has been returned, as the codec
    382  specific data may be lost during the flush. You must resubmit the data using buffers marked with
    383  {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation.
    384  <p>
    385  Encoders (or codecs that generate compressed data) will create and return the codec specific data
    386  before any valid output buffer in output buffers marked with the {@linkplain
    387  #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no
    388  meaningful timestamps.
    389 
    390  <h3>Data Processing</h3>
    391  <p>
    392  Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in
    393  API calls. After a successful call to {@link #start} the client "owns" neither input nor output
    394  buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link
    395  #dequeueOutputBuffer OutputBuffer(&hellip;)} to obtain (get ownership of) an input or output
    396  buffer from the codec. In asynchronous mode, you will automatically receive available buffers via
    397  the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link
    398  Callback#onOutputBufferAvailable OutputBufferAvailable(&hellip;)} callbacks.
    399  <p>
    400  Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link
    401  #queueInputBuffer queueInputBuffer} &ndash; or {@link #queueSecureInputBuffer
    402  queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same
    403  timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such).
    404  <p>
    405  The codec in turn will return a read-only output buffer via the {@link
    406  Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in
    407  response to a {@link #dequeueOutputBuffer dequeuOutputBuffer} call in synchronous mode. After the
    408  output buffer has been processed, call one of the {@link #releaseOutputBuffer
    409  releaseOutputBuffer} methods to return the buffer to the codec.
    410  <p>
    411  While you are not required to resubmit/release buffers immediately to the codec, holding onto
    412  input and/or output buffers may stall the codec, and this behavior is device dependent.
    413  <strong>Specifically, it is possible that a codec may hold off on generating output buffers until
    414  <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to
    415  hold onto to available buffers as little as possible.
    416  <p>
    417  Depending on the API version, you can process data in three ways:
    418  <table>
    419   <thead>
    420    <tr>
    421     <th>Processing Mode</th>
    422     <th>API version <= 20<br>Jelly Bean/KitKat</th>
    423     <th>API version >= 21<br>Lollipop and later</th>
    424    </tr>
    425   </thead>
    426   <tbody>
    427    <tr>
    428     <td>Synchronous API using buffer arrays</td>
    429     <td>Supported</td>
    430     <td>Deprecated</td>
    431    </tr>
    432    <tr>
    433     <td>Synchronous API using buffers</td>
    434     <td class=NA>Not Available</td>
    435     <td>Supported</td>
    436    </tr>
    437    <tr>
    438     <td>Asynchronous API using buffers</td>
    439     <td class=NA>Not Available</td>
    440     <td>Supported</td>
    441    </tr>
    442   </tbody>
    443  </table>
    444 
    445  <h4>Asynchronous Processing using Buffers</h4>
    446  <p>
    447  Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data
    448  asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous
    449  mode changes the state transitions slightly, because you must call {@link #start} after {@link
    450  #flush} to transition the codec to the Running sub-state and start receiving input buffers.
    451  Similarly, upon an initial call to {@code start} the codec will move directly to the Running
    452  sub-state and start passing available input buffers via the callback.
    453  <p>
    454  <center><object style="width: 516px; height: 353px;" type="image/svg+xml"
    455    data="../../../images/media/mediacodec_async_states.svg"><img
    456    src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px"
    457    alt="MediaCodec state diagram for asynchronous operation"></object></center>
    458  <p>
    459  MediaCodec is typically used like this in asynchronous mode:
    460  <pre class=prettyprint>
    461  MediaCodec codec = MediaCodec.createByCodecName(name);
    462  MediaFormat mOutputFormat; // member variable
    463  codec.setCallback(new MediaCodec.Callback() {
    464    {@literal @Override}
    465    void onInputBufferAvailable(MediaCodec mc, int inputBufferId) {
    466      ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId);
    467      // fill inputBuffer with valid data
    468      &hellip;
    469      codec.queueInputBuffer(inputBufferId, &hellip;);
    470    }
    471 
    472    {@literal @Override}
    473    void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, &hellip;) {
    474      ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
    475      MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A
    476      // bufferFormat is equivalent to mOutputFormat
    477      // outputBuffer is ready to be processed or rendered.
    478      &hellip;
    479      codec.releaseOutputBuffer(outputBufferId, &hellip;);
    480    }
    481 
    482    {@literal @Override}
    483    void onOutputFormatChanged(MediaCodec mc, MediaFormat format) {
    484      // Subsequent data will conform to new format.
    485      // Can ignore if using getOutputFormat(outputBufferId)
    486      mOutputFormat = format; // option B
    487    }
    488 
    489    {@literal @Override}
    490    void onError(&hellip;) {
    491      &hellip;
    492    }
    493  });
    494  codec.configure(format, &hellip;);
    495  mOutputFormat = codec.getOutputFormat(); // option B
    496  codec.start();
    497  // wait for processing to complete
    498  codec.stop();
    499  codec.release();</pre>
    500 
    501  <h4>Synchronous Processing using Buffers</h4>
    502  <p>
    503  Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output
    504  buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or
    505  {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the
    506  codec in synchronous mode. This allows certain optimizations by the framework, e.g. when
    507  processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers
    508  getInput}/{@link #getOutputBuffers OutputBuffers()}.
    509 
    510  <p class=note>
    511  <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same
    512  time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link
    513  #start} or after having dequeued an output buffer ID with the value of {@link
    514  #INFO_OUTPUT_FORMAT_CHANGED}.
    515  <p>
    516  MediaCodec is typically used like this in synchronous mode:
    517  <pre>
    518  MediaCodec codec = MediaCodec.createByCodecName(name);
    519  codec.configure(format, &hellip;);
    520  MediaFormat outputFormat = codec.getOutputFormat(); // option B
    521  codec.start();
    522  for (;;) {
    523    int inputBufferId = codec.dequeueInputBuffer(timeoutUs);
    524    if (inputBufferId &gt;= 0) {
    525      ByteBuffer inputBuffer = codec.getInputBuffer(&hellip;);
    526      // fill inputBuffer with valid data
    527      &hellip;
    528      codec.queueInputBuffer(inputBufferId, &hellip;);
    529    }
    530    int outputBufferId = codec.dequeueOutputBuffer(&hellip;);
    531    if (outputBufferId &gt;= 0) {
    532      ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
    533      MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A
    534      // bufferFormat is identical to outputFormat
    535      // outputBuffer is ready to be processed or rendered.
    536      &hellip;
    537      codec.releaseOutputBuffer(outputBufferId, &hellip;);
    538    } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
    539      // Subsequent data will conform to new format.
    540      // Can ignore if using getOutputFormat(outputBufferId)
    541      outputFormat = codec.getOutputFormat(); // option B
    542    }
    543  }
    544  codec.stop();
    545  codec.release();</pre>
    546 
    547  <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4>
    548  <p>
    549  In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and
    550  output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to
    551  {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link
    552  #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when
    553  non-negative), as demonstrated in the sample below. Note that there is no inherent correlation
    554  between the size of the arrays and the number of input and output buffers used by the system,
    555  although the array size provides an upper bound.
    556  <pre>
    557  MediaCodec codec = MediaCodec.createByCodecName(name);
    558  codec.configure(format, &hellip;);
    559  codec.start();
    560  ByteBuffer[] inputBuffers = codec.getInputBuffers();
    561  ByteBuffer[] outputBuffers = codec.getOutputBuffers();
    562  for (;;) {
    563    int inputBufferId = codec.dequeueInputBuffer(&hellip;);
    564    if (inputBufferId &gt;= 0) {
    565      // fill inputBuffers[inputBufferId] with valid data
    566      &hellip;
    567      codec.queueInputBuffer(inputBufferId, &hellip;);
    568    }
    569    int outputBufferId = codec.dequeueOutputBuffer(&hellip;);
    570    if (outputBufferId &gt;= 0) {
    571      // outputBuffers[outputBufferId] is ready to be processed or rendered.
    572      &hellip;
    573      codec.releaseOutputBuffer(outputBufferId, &hellip;);
    574    } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
    575      outputBuffers = codec.getOutputBuffers();
    576    } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
    577      // Subsequent data will conform to new format.
    578      MediaFormat format = codec.getOutputFormat();
    579    }
    580  }
    581  codec.stop();
    582  codec.release();</pre>
    583 
    584  <h4>End-of-stream Handling</h4>
    585  <p>
    586  When you reach the end of the input data, you must signal it to the codec by specifying the
    587  {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer
    588  queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional
    589  empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will
    590  be ignored.
    591  <p>
    592  The codec will continue to return output buffers until it eventually signals the end of the
    593  output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link
    594  #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable
    595  onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer
    596  after the last valid output buffer. The timestamp of such empty buffer should be ignored.
    597  <p>
    598  Do not submit additional input buffers after signaling the end of the input stream, unless the
    599  codec has been flushed, or stopped and restarted.
    600 
    601  <h4>Using an Output Surface</h4>
    602  <p>
    603  The data processing is nearly identical to the ByteBuffer mode when using an output {@link
    604  Surface}; however, the output buffers will not be accessible, and are represented as {@code null}
    605  values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will
    606  return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code
    607  null}-s.
    608  <p>
    609  When using an output Surface, you can select whether or not to render each output buffer on the
    610  surface. You have three choices:
    611  <ul>
    612  <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean)
    613  releaseOutputBuffer(bufferId, false)}.</li>
    614  <li><strong>Render the buffer with the default timestamp:</strong> Call {@link
    615  #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li>
    616  <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link
    617  #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li>
    618  </ul>
    619  <p>
    620  Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain
    621  BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds).
    622  It was not defined prior to that.
    623  <p>
    624  Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface
    625  dynamically using {@link #setOutputSurface setOutputSurface}.
    626 
    627  <h4>Transformations When Rendering onto Surface</h4>
    628 
    629  If the codec is configured into Surface mode, any crop rectangle, {@linkplain
    630  MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling
    631  mode} will be automatically applied with one exception:
    632  <p class=note>
    633  Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not
    634  have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard
    635  and simple way to identify software decoders, or if they apply the rotation other than by trying
    636  it out.
    637  <p>
    638  There are also some caveats.
    639  <p class=note>
    640  Note that the pixel aspect ratio is not considered when displaying the output onto the
    641  Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you
    642  must position the output Surface so that it has the proper final display aspect ratio. Conversely,
    643  you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with
    644  square pixels (pixel aspect ratio or 1:1).
    645  <p class=note>
    646  Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link
    647  #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated
    648  by 90 or 270 degrees.
    649  <p class=note>
    650  When setting the video scaling mode, note that it must be reset after each time the output
    651  buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can
    652  do this after each time the output format changes.
    653 
    654  <h4>Using an Input Surface</h4>
    655  <p>
    656  When using an input Surface, there are no accessible input buffers, as buffers are automatically
    657  passed from the input surface to the codec. Calling {@link #dequeueInputBuffer
    658  dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers}
    659  returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into.
    660  <p>
    661  Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop
    662  submitting data to the codec immediately after this call.
    663  <p>
    664 
    665  <h3>Seeking &amp; Adaptive Playback Support</h3>
    666  <p>
    667  Video decoders (and in general codecs that consume compressed video data) behave differently
    668  regarding seek and format change whether or not they support and are configured for adaptive
    669  playback. You can check if a decoder supports {@linkplain
    670  CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link
    671  CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive
    672  playback support for video decoders is only activated if you configure the codec to decode onto a
    673  {@link Surface}.
    674 
    675  <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4>
    676  <p>
    677  It is important that the input data after {@link #start} or {@link #flush} starts at a suitable
    678  stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded
    679  completely on its own (for most codecs this means an I-frame), and no frames that are to be
    680  displayed after a key frame refer to frames before the key frame.
    681  <p>
    682  The following table summarizes suitable key frames for various video formats.
    683  <table>
    684   <thead>
    685    <tr>
    686     <th>Format</th>
    687     <th>Suitable key frame</th>
    688    </tr>
    689   </thead>
    690   <tbody class=mid>
    691    <tr>
    692     <td>VP9/VP8</td>
    693     <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br>
    694       <i>(There is no specific name for such key frame.)</i></td>
    695    </tr>
    696    <tr>
    697     <td>H.265 HEVC</td>
    698     <td>IDR or CRA</td>
    699    </tr>
    700    <tr>
    701     <td>H.264 AVC</td>
    702     <td>IDR</td>
    703    </tr>
    704    <tr>
    705     <td>MPEG-4<br>H.263<br>MPEG-2</td>
    706     <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br>
    707       <i>(There is no specific name for such key frame.)</td>
    708    </tr>
    709   </tbody>
    710  </table>
    711 
    712  <h4>For decoders that do not support adaptive playback (including when not decoding onto a
    713  Surface)</h4>
    714  <p>
    715  In order to start decoding data that is not adjacent to previously submitted data (i.e. after a
    716  seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately
    717  revoked at the point of the flush, you may want to first signal then wait for the end-of-stream
    718  before you call {@code flush}. It is important that the input data after a flush starts at a
    719  suitable stream boundary/key frame.
    720  <p class=note>
    721  <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link
    722  #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link
    723  #configure configure(&hellip;)} - {@link #start} cycle is necessary.
    724 
    725  <p class=note>
    726  <strong>Also note:</strong> if you flush the codec too soon after {@link #start} &ndash;
    727  generally, before the first output buffer or output format change is received &ndash; you
    728  will need to resubmit the codec-specific-data to the codec. See the <a
    729  href="#CSD">codec-specific-data section</a> for more info.
    730 
    731  <h4>For decoders that support and are configured for adaptive playback</h4>
    732  <p>
    733  In order to start decoding data that is not adjacent to previously submitted data (i.e. after a
    734  seek) it is <em>not necessary</em> to flush the decoder; however, input data after the
    735  discontinuity must start at a suitable stream boundary/key frame.
    736  <p>
    737  For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the
    738  picture size or configuration mid-stream. To do this you must package the entire new
    739  codec-specific configuration data together with the key frame into a single buffer (including
    740  any start codes), and submit it as a <strong>regular</strong> input buffer.
    741  <p>
    742  You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link
    743  #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable
    744  onOutputFormatChanged} callback just after the picture-size change takes place and before any
    745  frames with the new size have been returned.
    746  <p class=note>
    747  <strong>Note:</strong> just as the case for codec-specific data, be careful when calling
    748  {@link #flush} shortly after you have changed the picture size. If you have not received
    749  confirmation of the picture size change, you will need to repeat the request for the new picture
    750  size.
    751 
    752  <h3>Error handling</h3>
    753  <p>
    754  The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType
    755  createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure
    756  which you must catch or declare to pass up. MediaCodec methods throw {@code
    757  IllegalStateException} when the method is called from a codec state that does not allow it; this
    758  is typically due to incorrect application API usage. Methods involving secure buffers may throw
    759  {@link CryptoException}, which has further error information obtainable from {@link
    760  CryptoException#getErrorCode}.
    761  <p>
    762  Internal codec errors result in a {@link CodecException}, which may be due to media content
    763  corruption, hardware failure, resource exhaustion, and so forth, even when the application is
    764  correctly using the API. The recommended action when receiving a {@code CodecException}
    765  can be determined by calling {@link CodecException#isRecoverable} and {@link
    766  CodecException#isTransient}:
    767  <ul>
    768  <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call
    769  {@link #stop}, {@link #configure configure(&hellip;)}, and {@link #start} to recover.</li>
    770  <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are
    771  temporarily unavailable and the method may be retried at a later time.</li>
    772  <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()}
    773  return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset
    774  reset} or {@linkplain #release released}.</li>
    775  </ul>
    776  <p>
    777  Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time.
    778 
    779  <h2 id=History><a name="History"></a>Valid API Calls and API History</h2>
    780  <p>
    781  This sections summarizes the valid API calls in each state and the API history of the MediaCodec
    782  class. For API version numbers, see {@link android.os.Build.VERSION_CODES}.
    783 
    784  <style>
    785  .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; }
    786  .api > tr > th     { vertical-align: bottom; }
    787  .api > tr > td     { vertical-align: middle; }
    788  .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; }
    789  .fn { text-align: left; }
    790  .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; }
    791  .deg45 {
    792    white-space: nowrap; background: none; border: none; vertical-align: bottom;
    793    width: 30px; height: 83px;
    794  }
    795  .deg45 > div {
    796    transform: skew(-45deg, 0deg) translate(1px, -67px);
    797    transform-origin: bottom left 0;
    798    width: 30px; height: 20px;
    799  }
    800  .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; }
    801  .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); }
    802  </style>
    803 
    804  <table align="right" style="width: 0%">
    805   <thead>
    806    <tr><th>Symbol</th><th>Meaning</th></tr>
    807   </thead>
    808   <tbody class=sml>
    809    <tr><td>&#9679;</td><td>Supported</td></tr>
    810    <tr><td>&#8277;</td><td>Semantics changed</td></tr>
    811    <tr><td>&#9675;</td><td>Experimental support</td></tr>
    812    <tr><td>[ ]</td><td>Deprecated</td></tr>
    813    <tr><td>&#9099;</td><td>Restricted to surface input mode</td></tr>
    814    <tr><td>&#9094;</td><td>Restricted to surface output mode</td></tr>
    815    <tr><td>&#9639;</td><td>Restricted to ByteBuffer input mode</td></tr>
    816    <tr><td>&#8617;</td><td>Restricted to synchronous mode</td></tr>
    817    <tr><td>&#8644;</td><td>Restricted to asynchronous mode</td></tr>
    818    <tr><td>( )</td><td>Can be called, but shouldn't</td></tr>
    819   </tbody>
    820  </table>
    821 
    822  <table style="width: 100%;">
    823   <thead class=api>
    824    <tr>
    825     <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th>
    826     <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th>
    827     <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th>
    828     <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th>
    829     <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th>
    830     <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th>
    831     <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th>
    832     <th></th>
    833     <th colspan="8">SDK Version</th>
    834    </tr>
    835    <tr>
    836     <th colspan="7">State</th>
    837     <th>Method</th>
    838     <th>16</th>
    839     <th>17</th>
    840     <th>18</th>
    841     <th>19</th>
    842     <th>20</th>
    843     <th>21</th>
    844     <th>22</th>
    845     <th>23</th>
    846    </tr>
    847   </thead>
    848   <tbody class=api>
    849    <tr>
    850     <td></td>
    851     <td></td>
    852     <td></td>
    853     <td></td>
    854     <td></td>
    855     <td></td>
    856     <td></td>
    857     <td class=fn>{@link #createByCodecName createByCodecName}</td>
    858     <td>&#9679;</td>
    859     <td>&#9679;</td>
    860     <td>&#9679;</td>
    861     <td>&#9679;</td>
    862     <td>&#9679;</td>
    863     <td>&#9679;</td>
    864     <td>&#9679;</td>
    865     <td>&#9679;</td>
    866    </tr>
    867    <tr>
    868     <td></td>
    869     <td></td>
    870     <td></td>
    871     <td></td>
    872     <td></td>
    873     <td></td>
    874     <td></td>
    875     <td class=fn>{@link #createDecoderByType createDecoderByType}</td>
    876     <td>&#9679;</td>
    877     <td>&#9679;</td>
    878     <td>&#9679;</td>
    879     <td>&#9679;</td>
    880     <td>&#9679;</td>
    881     <td>&#9679;</td>
    882     <td>&#9679;</td>
    883     <td>&#9679;</td>
    884    </tr>
    885    <tr>
    886     <td></td>
    887     <td></td>
    888     <td></td>
    889     <td></td>
    890     <td></td>
    891     <td></td>
    892     <td></td>
    893     <td class=fn>{@link #createEncoderByType createEncoderByType}</td>
    894     <td>&#9679;</td>
    895     <td>&#9679;</td>
    896     <td>&#9679;</td>
    897     <td>&#9679;</td>
    898     <td>&#9679;</td>
    899     <td>&#9679;</td>
    900     <td>&#9679;</td>
    901     <td>&#9679;</td>
    902    </tr>
    903    <tr>
    904     <td></td>
    905     <td></td>
    906     <td></td>
    907     <td></td>
    908     <td></td>
    909     <td></td>
    910     <td></td>
    911     <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td>
    912     <td></td>
    913     <td></td>
    914     <td></td>
    915     <td></td>
    916     <td></td>
    917     <td></td>
    918     <td></td>
    919     <td>&#9679;</td>
    920    </tr>
    921    <tr>
    922     <td>16+</td>
    923     <td>-</td>
    924     <td>-</td>
    925     <td>-</td>
    926     <td>-</td>
    927     <td>-</td>
    928     <td>-</td>
    929     <td class=fn>{@link #configure configure}</td>
    930     <td>&#9679;</td>
    931     <td>&#9679;</td>
    932     <td>&#9679;</td>
    933     <td>&#9679;</td>
    934     <td>&#9679;</td>
    935     <td>&#8277;</td>
    936     <td>&#9679;</td>
    937     <td>&#9679;</td>
    938    </tr>
    939    <tr>
    940     <td>-</td>
    941     <td>18+</td>
    942     <td>-</td>
    943     <td>-</td>
    944     <td>-</td>
    945     <td>-</td>
    946     <td>-</td>
    947     <td class=fn>{@link #createInputSurface createInputSurface}</td>
    948     <td></td>
    949     <td></td>
    950     <td>&#9099;</td>
    951     <td>&#9099;</td>
    952     <td>&#9099;</td>
    953     <td>&#9099;</td>
    954     <td>&#9099;</td>
    955     <td>&#9099;</td>
    956    </tr>
    957    <tr>
    958     <td>-</td>
    959     <td>-</td>
    960     <td>16+</td>
    961     <td>16+</td>
    962     <td>(16+)</td>
    963     <td>-</td>
    964     <td>-</td>
    965     <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td>
    966     <td>&#9679;</td>
    967     <td>&#9679;</td>
    968     <td>&#9639;</td>
    969     <td>&#9639;</td>
    970     <td>&#9639;</td>
    971     <td>&#8277;&#9639;&#8617;</td>
    972     <td>&#9639;&#8617;</td>
    973     <td>&#9639;&#8617;</td>
    974    </tr>
    975    <tr>
    976     <td>-</td>
    977     <td>-</td>
    978     <td>16+</td>
    979     <td>16+</td>
    980     <td>16+</td>
    981     <td>-</td>
    982     <td>-</td>
    983     <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td>
    984     <td>&#9679;</td>
    985     <td>&#9679;</td>
    986     <td>&#9679;</td>
    987     <td>&#9679;</td>
    988     <td>&#9679;</td>
    989     <td>&#8277;&#8617;</td>
    990     <td>&#8617;</td>
    991     <td>&#8617;</td>
    992    </tr>
    993    <tr>
    994     <td>-</td>
    995     <td>-</td>
    996     <td>16+</td>
    997     <td>16+</td>
    998     <td>16+</td>
    999     <td>-</td>
   1000     <td>-</td>
   1001     <td class=fn>{@link #flush flush}</td>
   1002     <td>&#9679;</td>
   1003     <td>&#9679;</td>
   1004     <td>&#9679;</td>
   1005     <td>&#9679;</td>
   1006     <td>&#9679;</td>
   1007     <td>&#9679;</td>
   1008     <td>&#9679;</td>
   1009     <td>&#9679;</td>
   1010    </tr>
   1011    <tr>
   1012     <td>18+</td>
   1013     <td>18+</td>
   1014     <td>18+</td>
   1015     <td>18+</td>
   1016     <td>18+</td>
   1017     <td>18+</td>
   1018     <td>-</td>
   1019     <td class=fn>{@link #getCodecInfo getCodecInfo}</td>
   1020     <td></td>
   1021     <td></td>
   1022     <td>&#9679;</td>
   1023     <td>&#9679;</td>
   1024     <td>&#9679;</td>
   1025     <td>&#9679;</td>
   1026     <td>&#9679;</td>
   1027     <td>&#9679;</td>
   1028    </tr>
   1029    <tr>
   1030     <td>-</td>
   1031     <td>-</td>
   1032     <td>(21+)</td>
   1033     <td>21+</td>
   1034     <td>(21+)</td>
   1035     <td>-</td>
   1036     <td>-</td>
   1037     <td class=fn>{@link #getInputBuffer getInputBuffer}</td>
   1038     <td></td>
   1039     <td></td>
   1040     <td></td>
   1041     <td></td>
   1042     <td></td>
   1043     <td>&#9679;</td>
   1044     <td>&#9679;</td>
   1045     <td>&#9679;</td>
   1046    </tr>
   1047    <tr>
   1048     <td>-</td>
   1049     <td>-</td>
   1050     <td>16+</td>
   1051     <td>(16+)</td>
   1052     <td>(16+)</td>
   1053     <td>-</td>
   1054     <td>-</td>
   1055     <td class=fn>{@link #getInputBuffers getInputBuffers}</td>
   1056     <td>&#9679;</td>
   1057     <td>&#9679;</td>
   1058     <td>&#9679;</td>
   1059     <td>&#9679;</td>
   1060     <td>&#9679;</td>
   1061     <td>[&#8277;&#8617;]</td>
   1062     <td>[&#8617;]</td>
   1063     <td>[&#8617;]</td>
   1064    </tr>
   1065    <tr>
   1066     <td>-</td>
   1067     <td>21+</td>
   1068     <td>(21+)</td>
   1069     <td>(21+)</td>
   1070     <td>(21+)</td>
   1071     <td>-</td>
   1072     <td>-</td>
   1073     <td class=fn>{@link #getInputFormat getInputFormat}</td>
   1074     <td></td>
   1075     <td></td>
   1076     <td></td>
   1077     <td></td>
   1078     <td></td>
   1079     <td>&#9679;</td>
   1080     <td>&#9679;</td>
   1081     <td>&#9679;</td>
   1082    </tr>
   1083    <tr>
   1084     <td>-</td>
   1085     <td>-</td>
   1086     <td>(21+)</td>
   1087     <td>21+</td>
   1088     <td>(21+)</td>
   1089     <td>-</td>
   1090     <td>-</td>
   1091     <td class=fn>{@link #getInputImage getInputImage}</td>
   1092     <td></td>
   1093     <td></td>
   1094     <td></td>
   1095     <td></td>
   1096     <td></td>
   1097     <td>&#9675;</td>
   1098     <td>&#9679;</td>
   1099     <td>&#9679;</td>
   1100    </tr>
   1101    <tr>
   1102     <td>18+</td>
   1103     <td>18+</td>
   1104     <td>18+</td>
   1105     <td>18+</td>
   1106     <td>18+</td>
   1107     <td>18+</td>
   1108     <td>-</td>
   1109     <td class=fn>{@link #getName getName}</td>
   1110     <td></td>
   1111     <td></td>
   1112     <td>&#9679;</td>
   1113     <td>&#9679;</td>
   1114     <td>&#9679;</td>
   1115     <td>&#9679;</td>
   1116     <td>&#9679;</td>
   1117     <td>&#9679;</td>
   1118    </tr>
   1119    <tr>
   1120     <td>-</td>
   1121     <td>-</td>
   1122     <td>(21+)</td>
   1123     <td>21+</td>
   1124     <td>21+</td>
   1125     <td>-</td>
   1126     <td>-</td>
   1127     <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td>
   1128     <td></td>
   1129     <td></td>
   1130     <td></td>
   1131     <td></td>
   1132     <td></td>
   1133     <td>&#9679;</td>
   1134     <td>&#9679;</td>
   1135     <td>&#9679;</td>
   1136    </tr>
   1137    <tr>
   1138     <td>-</td>
   1139     <td>-</td>
   1140     <td>16+</td>
   1141     <td>16+</td>
   1142     <td>16+</td>
   1143     <td>-</td>
   1144     <td>-</td>
   1145     <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td>
   1146     <td>&#9679;</td>
   1147     <td>&#9679;</td>
   1148     <td>&#9679;</td>
   1149     <td>&#9679;</td>
   1150     <td>&#9679;</td>
   1151     <td>[&#8277;&#8617;]</td>
   1152     <td>[&#8617;]</td>
   1153     <td>[&#8617;]</td>
   1154    </tr>
   1155    <tr>
   1156     <td>-</td>
   1157     <td>21+</td>
   1158     <td>16+</td>
   1159     <td>16+</td>
   1160     <td>16+</td>
   1161     <td>-</td>
   1162     <td>-</td>
   1163     <td class=fn>{@link #getOutputFormat()}</td>
   1164     <td>&#9679;</td>
   1165     <td>&#9679;</td>
   1166     <td>&#9679;</td>
   1167     <td>&#9679;</td>
   1168     <td>&#9679;</td>
   1169     <td>&#9679;</td>
   1170     <td>&#9679;</td>
   1171     <td>&#9679;</td>
   1172    </tr>
   1173    <tr>
   1174     <td>-</td>
   1175     <td>-</td>
   1176     <td>(21+)</td>
   1177     <td>21+</td>
   1178     <td>21+</td>
   1179     <td>-</td>
   1180     <td>-</td>
   1181     <td class=fn>{@link #getOutputFormat(int)}</td>
   1182     <td></td>
   1183     <td></td>
   1184     <td></td>
   1185     <td></td>
   1186     <td></td>
   1187     <td>&#9679;</td>
   1188     <td>&#9679;</td>
   1189     <td>&#9679;</td>
   1190    </tr>
   1191    <tr>
   1192     <td>-</td>
   1193     <td>-</td>
   1194     <td>(21+)</td>
   1195     <td>21+</td>
   1196     <td>21+</td>
   1197     <td>-</td>
   1198     <td>-</td>
   1199     <td class=fn>{@link #getOutputImage getOutputImage}</td>
   1200     <td></td>
   1201     <td></td>
   1202     <td></td>
   1203     <td></td>
   1204     <td></td>
   1205     <td>&#9675;</td>
   1206     <td>&#9679;</td>
   1207     <td>&#9679;</td>
   1208    </tr>
   1209    <tr>
   1210     <td>-</td>
   1211     <td>-</td>
   1212     <td>-</td>
   1213     <td>16+</td>
   1214     <td>(16+)</td>
   1215     <td>-</td>
   1216     <td>-</td>
   1217     <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td>
   1218     <td>&#9679;</td>
   1219     <td>&#9679;</td>
   1220     <td>&#9679;</td>
   1221     <td>&#9679;</td>
   1222     <td>&#9679;</td>
   1223     <td>&#8277;</td>
   1224     <td>&#9679;</td>
   1225     <td>&#9679;</td>
   1226    </tr>
   1227    <tr>
   1228     <td>-</td>
   1229     <td>-</td>
   1230     <td>-</td>
   1231     <td>16+</td>
   1232     <td>(16+)</td>
   1233     <td>-</td>
   1234     <td>-</td>
   1235     <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td>
   1236     <td>&#9679;</td>
   1237     <td>&#9679;</td>
   1238     <td>&#9679;</td>
   1239     <td>&#9679;</td>
   1240     <td>&#9679;</td>
   1241     <td>&#8277;</td>
   1242     <td>&#9679;</td>
   1243     <td>&#9679;</td>
   1244    </tr>
   1245    <tr>
   1246     <td>16+</td>
   1247     <td>16+</td>
   1248     <td>16+</td>
   1249     <td>16+</td>
   1250     <td>16+</td>
   1251     <td>16+</td>
   1252     <td>16+</td>
   1253     <td class=fn>{@link #release release}</td>
   1254     <td>&#9679;</td>
   1255     <td>&#9679;</td>
   1256     <td>&#9679;</td>
   1257     <td>&#9679;</td>
   1258     <td>&#9679;</td>
   1259     <td>&#9679;</td>
   1260     <td>&#9679;</td>
   1261     <td>&#9679;</td>
   1262    </tr>
   1263    <tr>
   1264     <td>-</td>
   1265     <td>-</td>
   1266     <td>-</td>
   1267     <td>16+</td>
   1268     <td>16+</td>
   1269     <td>-</td>
   1270     <td>-</td>
   1271     <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td>
   1272     <td>&#9679;</td>
   1273     <td>&#9679;</td>
   1274     <td>&#9679;</td>
   1275     <td>&#9679;</td>
   1276     <td>&#9679;</td>
   1277     <td>&#8277;</td>
   1278     <td>&#9679;</td>
   1279     <td>&#8277;</td>
   1280    </tr>
   1281    <tr>
   1282     <td>-</td>
   1283     <td>-</td>
   1284     <td>-</td>
   1285     <td>21+</td>
   1286     <td>21+</td>
   1287     <td>-</td>
   1288     <td>-</td>
   1289     <td class=fn>{@link #releaseOutputBuffer(int, long)}</td>
   1290     <td></td>
   1291     <td></td>
   1292     <td></td>
   1293     <td></td>
   1294     <td></td>
   1295     <td>&#9094;</td>
   1296     <td>&#9094;</td>
   1297     <td>&#9094;</td>
   1298    </tr>
   1299    <tr>
   1300     <td>21+</td>
   1301     <td>21+</td>
   1302     <td>21+</td>
   1303     <td>21+</td>
   1304     <td>21+</td>
   1305     <td>21+</td>
   1306     <td>-</td>
   1307     <td class=fn>{@link #reset reset}</td>
   1308     <td></td>
   1309     <td></td>
   1310     <td></td>
   1311     <td></td>
   1312     <td></td>
   1313     <td>&#9679;</td>
   1314     <td>&#9679;</td>
   1315     <td>&#9679;</td>
   1316    </tr>
   1317    <tr>
   1318     <td>21+</td>
   1319     <td>-</td>
   1320     <td>-</td>
   1321     <td>-</td>
   1322     <td>-</td>
   1323     <td>-</td>
   1324     <td>-</td>
   1325     <td class=fn>{@link #setCallback(Callback) setCallback}</td>
   1326     <td></td>
   1327     <td></td>
   1328     <td></td>
   1329     <td></td>
   1330     <td></td>
   1331     <td>&#9679;</td>
   1332     <td>&#9679;</td>
   1333     <td>{@link #setCallback(Callback, Handler) &#8277;}</td>
   1334    </tr>
   1335    <tr>
   1336     <td>-</td>
   1337     <td>23+</td>
   1338     <td>-</td>
   1339     <td>-</td>
   1340     <td>-</td>
   1341     <td>-</td>
   1342     <td>-</td>
   1343     <td class=fn>{@link #setInputSurface setInputSurface}</td>
   1344     <td></td>
   1345     <td></td>
   1346     <td></td>
   1347     <td></td>
   1348     <td></td>
   1349     <td></td>
   1350     <td></td>
   1351     <td>&#9099;</td>
   1352    </tr>
   1353    <tr>
   1354     <td>23+</td>
   1355     <td>23+</td>
   1356     <td>23+</td>
   1357     <td>23+</td>
   1358     <td>23+</td>
   1359     <td>(23+)</td>
   1360     <td>(23+)</td>
   1361     <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td>
   1362     <td></td>
   1363     <td></td>
   1364     <td></td>
   1365     <td></td>
   1366     <td></td>
   1367     <td></td>
   1368     <td></td>
   1369     <td>&#9675; &#9094;</td>
   1370    </tr>
   1371    <tr>
   1372     <td>-</td>
   1373     <td>23+</td>
   1374     <td>23+</td>
   1375     <td>23+</td>
   1376     <td>23+</td>
   1377     <td>-</td>
   1378     <td>-</td>
   1379     <td class=fn>{@link #setOutputSurface setOutputSurface}</td>
   1380     <td></td>
   1381     <td></td>
   1382     <td></td>
   1383     <td></td>
   1384     <td></td>
   1385     <td></td>
   1386     <td></td>
   1387     <td>&#9094;</td>
   1388    </tr>
   1389    <tr>
   1390     <td>19+</td>
   1391     <td>19+</td>
   1392     <td>19+</td>
   1393     <td>19+</td>
   1394     <td>19+</td>
   1395     <td>(19+)</td>
   1396     <td>-</td>
   1397     <td class=fn>{@link #setParameters setParameters}</td>
   1398     <td></td>
   1399     <td></td>
   1400     <td></td>
   1401     <td>&#9679;</td>
   1402     <td>&#9679;</td>
   1403     <td>&#9679;</td>
   1404     <td>&#9679;</td>
   1405     <td>&#9679;</td>
   1406    </tr>
   1407    <tr>
   1408     <td>-</td>
   1409     <td>(16+)</td>
   1410     <td>(16+)</td>
   1411     <td>16+</td>
   1412     <td>(16+)</td>
   1413     <td>(16+)</td>
   1414     <td>-</td>
   1415     <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td>
   1416     <td>&#9094;</td>
   1417     <td>&#9094;</td>
   1418     <td>&#9094;</td>
   1419     <td>&#9094;</td>
   1420     <td>&#9094;</td>
   1421     <td>&#9094;</td>
   1422     <td>&#9094;</td>
   1423     <td>&#9094;</td>
   1424    </tr>
   1425    <tr>
   1426     <td>-</td>
   1427     <td>-</td>
   1428     <td>18+</td>
   1429     <td>18+</td>
   1430     <td>-</td>
   1431     <td>-</td>
   1432     <td>-</td>
   1433     <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td>
   1434     <td></td>
   1435     <td></td>
   1436     <td>&#9099;</td>
   1437     <td>&#9099;</td>
   1438     <td>&#9099;</td>
   1439     <td>&#9099;</td>
   1440     <td>&#9099;</td>
   1441     <td>&#9099;</td>
   1442    </tr>
   1443    <tr>
   1444     <td>-</td>
   1445     <td>16+</td>
   1446     <td>21+(&#8644;)</td>
   1447     <td>-</td>
   1448     <td>-</td>
   1449     <td>-</td>
   1450     <td>-</td>
   1451     <td class=fn>{@link #start start}</td>
   1452     <td>&#9679;</td>
   1453     <td>&#9679;</td>
   1454     <td>&#9679;</td>
   1455     <td>&#9679;</td>
   1456     <td>&#9679;</td>
   1457     <td>&#8277;</td>
   1458     <td>&#9679;</td>
   1459     <td>&#9679;</td>
   1460    </tr>
   1461    <tr>
   1462     <td>-</td>
   1463     <td>-</td>
   1464     <td>16+</td>
   1465     <td>16+</td>
   1466     <td>16+</td>
   1467     <td>-</td>
   1468     <td>-</td>
   1469     <td class=fn>{@link #stop stop}</td>
   1470     <td>&#9679;</td>
   1471     <td>&#9679;</td>
   1472     <td>&#9679;</td>
   1473     <td>&#9679;</td>
   1474     <td>&#9679;</td>
   1475     <td>&#9679;</td>
   1476     <td>&#9679;</td>
   1477     <td>&#9679;</td>
   1478    </tr>
   1479   </tbody>
   1480  </table>
   1481  */
   1482 final public class MediaCodec {
   1483     /**
   1484      * Per buffer metadata includes an offset and size specifying
   1485      * the range of valid data in the associated codec (output) buffer.
   1486      */
   1487     public final static class BufferInfo {
   1488         /**
   1489          * Update the buffer metadata information.
   1490          *
   1491          * @param newOffset the start-offset of the data in the buffer.
   1492          * @param newSize   the amount of data (in bytes) in the buffer.
   1493          * @param newTimeUs the presentation timestamp in microseconds.
   1494          * @param newFlags  buffer flags associated with the buffer.  This
   1495          * should be a combination of  {@link #BUFFER_FLAG_KEY_FRAME} and
   1496          * {@link #BUFFER_FLAG_END_OF_STREAM}.
   1497          */
   1498         public void set(
   1499                 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) {
   1500             offset = newOffset;
   1501             size = newSize;
   1502             presentationTimeUs = newTimeUs;
   1503             flags = newFlags;
   1504         }
   1505 
   1506         /**
   1507          * The start-offset of the data in the buffer.
   1508          */
   1509         public int offset;
   1510 
   1511         /**
   1512          * The amount of data (in bytes) in the buffer.  If this is {@code 0},
   1513          * the buffer has no data in it and can be discarded.  The only
   1514          * use of a 0-size buffer is to carry the end-of-stream marker.
   1515          */
   1516         public int size;
   1517 
   1518         /**
   1519          * The presentation timestamp in microseconds for the buffer.
   1520          * This is derived from the presentation timestamp passed in
   1521          * with the corresponding input buffer.  This should be ignored for
   1522          * a 0-sized buffer.
   1523          */
   1524         public long presentationTimeUs;
   1525 
   1526         /**
   1527          * Buffer flags associated with the buffer.  A combination of
   1528          * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}.
   1529          *
   1530          * <p>Encoded buffers that are key frames are marked with
   1531          * {@link #BUFFER_FLAG_KEY_FRAME}.
   1532          *
   1533          * <p>The last output buffer corresponding to the input buffer
   1534          * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked
   1535          * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could
   1536          * be an empty buffer, whose sole purpose is to carry the end-of-stream
   1537          * marker.
   1538          */
   1539         @BufferFlag
   1540         public int flags;
   1541 
   1542         /** @hide */
   1543         @NonNull
   1544         public BufferInfo dup() {
   1545             BufferInfo copy = new BufferInfo();
   1546             copy.set(offset, size, presentationTimeUs, flags);
   1547             return copy;
   1548         }
   1549     };
   1550 
   1551     // The follow flag constants MUST stay in sync with their equivalents
   1552     // in MediaCodec.h !
   1553 
   1554     /**
   1555      * This indicates that the (encoded) buffer marked as such contains
   1556      * the data for a key frame.
   1557      *
   1558      * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead.
   1559      */
   1560     public static final int BUFFER_FLAG_SYNC_FRAME = 1;
   1561 
   1562     /**
   1563      * This indicates that the (encoded) buffer marked as such contains
   1564      * the data for a key frame.
   1565      */
   1566     public static final int BUFFER_FLAG_KEY_FRAME = 1;
   1567 
   1568     /**
   1569      * This indicated that the buffer marked as such contains codec
   1570      * initialization / codec specific data instead of media data.
   1571      */
   1572     public static final int BUFFER_FLAG_CODEC_CONFIG = 2;
   1573 
   1574     /**
   1575      * This signals the end of stream, i.e. no buffers will be available
   1576      * after this, unless of course, {@link #flush} follows.
   1577      */
   1578     public static final int BUFFER_FLAG_END_OF_STREAM = 4;
   1579 
   1580     /**
   1581      * This indicates that the buffer only contains part of a frame,
   1582      * and the decoder should batch the data until a buffer without
   1583      * this flag appears before decoding the frame.
   1584      */
   1585     public static final int BUFFER_FLAG_PARTIAL_FRAME = 8;
   1586 
   1587     /** @hide */
   1588     @IntDef(
   1589         flag = true,
   1590         value = {
   1591             BUFFER_FLAG_SYNC_FRAME,
   1592             BUFFER_FLAG_KEY_FRAME,
   1593             BUFFER_FLAG_CODEC_CONFIG,
   1594             BUFFER_FLAG_END_OF_STREAM,
   1595             BUFFER_FLAG_PARTIAL_FRAME,
   1596     })
   1597     @Retention(RetentionPolicy.SOURCE)
   1598     public @interface BufferFlag {}
   1599 
   1600     private EventHandler mEventHandler;
   1601     private EventHandler mOnFrameRenderedHandler;
   1602     private EventHandler mCallbackHandler;
   1603     private Callback mCallback;
   1604     private OnFrameRenderedListener mOnFrameRenderedListener;
   1605     private Object mListenerLock = new Object();
   1606 
   1607     private static final int EVENT_CALLBACK = 1;
   1608     private static final int EVENT_SET_CALLBACK = 2;
   1609     private static final int EVENT_FRAME_RENDERED = 3;
   1610 
   1611     private static final int CB_INPUT_AVAILABLE = 1;
   1612     private static final int CB_OUTPUT_AVAILABLE = 2;
   1613     private static final int CB_ERROR = 3;
   1614     private static final int CB_OUTPUT_FORMAT_CHANGE = 4;
   1615 
   1616     private class EventHandler extends Handler {
   1617         private MediaCodec mCodec;
   1618 
   1619         public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) {
   1620             super(looper);
   1621             mCodec = codec;
   1622         }
   1623 
   1624         @Override
   1625         public void handleMessage(@NonNull Message msg) {
   1626             switch (msg.what) {
   1627                 case EVENT_CALLBACK:
   1628                 {
   1629                     handleCallback(msg);
   1630                     break;
   1631                 }
   1632                 case EVENT_SET_CALLBACK:
   1633                 {
   1634                     mCallback = (MediaCodec.Callback) msg.obj;
   1635                     break;
   1636                 }
   1637                 case EVENT_FRAME_RENDERED:
   1638                     synchronized (mListenerLock) {
   1639                         Map<String, Object> map = (Map<String, Object>)msg.obj;
   1640                         for (int i = 0; ; ++i) {
   1641                             Object mediaTimeUs = map.get(i + "-media-time-us");
   1642                             Object systemNano = map.get(i + "-system-nano");
   1643                             if (mediaTimeUs == null || systemNano == null
   1644                                     || mOnFrameRenderedListener == null) {
   1645                                 break;
   1646                             }
   1647                             mOnFrameRenderedListener.onFrameRendered(
   1648                                     mCodec, (long)mediaTimeUs, (long)systemNano);
   1649                         }
   1650                         break;
   1651                     }
   1652                 default:
   1653                 {
   1654                     break;
   1655                 }
   1656             }
   1657         }
   1658 
   1659         private void handleCallback(@NonNull Message msg) {
   1660             if (mCallback == null) {
   1661                 return;
   1662             }
   1663 
   1664             switch (msg.arg1) {
   1665                 case CB_INPUT_AVAILABLE:
   1666                 {
   1667                     int index = msg.arg2;
   1668                     synchronized(mBufferLock) {
   1669                         validateInputByteBuffer(mCachedInputBuffers, index);
   1670                     }
   1671                     mCallback.onInputBufferAvailable(mCodec, index);
   1672                     break;
   1673                 }
   1674 
   1675                 case CB_OUTPUT_AVAILABLE:
   1676                 {
   1677                     int index = msg.arg2;
   1678                     BufferInfo info = (MediaCodec.BufferInfo) msg.obj;
   1679                     synchronized(mBufferLock) {
   1680                         validateOutputByteBuffer(mCachedOutputBuffers, index, info);
   1681                     }
   1682                     mCallback.onOutputBufferAvailable(
   1683                             mCodec, index, info);
   1684                     break;
   1685                 }
   1686 
   1687                 case CB_ERROR:
   1688                 {
   1689                     mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj);
   1690                     break;
   1691                 }
   1692 
   1693                 case CB_OUTPUT_FORMAT_CHANGE:
   1694                 {
   1695                     mCallback.onOutputFormatChanged(mCodec,
   1696                             new MediaFormat((Map<String, Object>) msg.obj));
   1697                     break;
   1698                 }
   1699 
   1700                 default:
   1701                 {
   1702                     break;
   1703                 }
   1704             }
   1705         }
   1706     }
   1707 
   1708     private boolean mHasSurface = false;
   1709 
   1710     /**
   1711      * Instantiate the preferred decoder supporting input data of the given mime type.
   1712      *
   1713      * The following is a partial list of defined mime types and their semantics:
   1714      * <ul>
   1715      * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm)
   1716      * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm)
   1717      * <li>"video/avc" - H.264/AVC video
   1718      * <li>"video/hevc" - H.265/HEVC video
   1719      * <li>"video/mp4v-es" - MPEG4 video
   1720      * <li>"video/3gpp" - H.263 video
   1721      * <li>"audio/3gpp" - AMR narrowband audio
   1722      * <li>"audio/amr-wb" - AMR wideband audio
   1723      * <li>"audio/mpeg" - MPEG1/2 audio layer III
   1724      * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!)
   1725      * <li>"audio/vorbis" - vorbis audio
   1726      * <li>"audio/g711-alaw" - G.711 alaw audio
   1727      * <li>"audio/g711-mlaw" - G.711 ulaw audio
   1728      * </ul>
   1729      *
   1730      * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat}
   1731      * and {@link #createByCodecName} to ensure that the resulting codec can handle a
   1732      * given format.
   1733      *
   1734      * @param type The mime type of the input data.
   1735      * @throws IOException if the codec cannot be created.
   1736      * @throws IllegalArgumentException if type is not a valid mime type.
   1737      * @throws NullPointerException if type is null.
   1738      */
   1739     @NonNull
   1740     public static MediaCodec createDecoderByType(@NonNull String type)
   1741             throws IOException {
   1742         return new MediaCodec(type, true /* nameIsType */, false /* encoder */);
   1743     }
   1744 
   1745     /**
   1746      * Instantiate the preferred encoder supporting output data of the given mime type.
   1747      *
   1748      * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat}
   1749      * and {@link #createByCodecName} to ensure that the resulting codec can handle a
   1750      * given format.
   1751      *
   1752      * @param type The desired mime type of the output data.
   1753      * @throws IOException if the codec cannot be created.
   1754      * @throws IllegalArgumentException if type is not a valid mime type.
   1755      * @throws NullPointerException if type is null.
   1756      */
   1757     @NonNull
   1758     public static MediaCodec createEncoderByType(@NonNull String type)
   1759             throws IOException {
   1760         return new MediaCodec(type, true /* nameIsType */, true /* encoder */);
   1761     }
   1762 
   1763     /**
   1764      * If you know the exact name of the component you want to instantiate
   1765      * use this method to instantiate it. Use with caution.
   1766      * Likely to be used with information obtained from {@link android.media.MediaCodecList}
   1767      * @param name The name of the codec to be instantiated.
   1768      * @throws IOException if the codec cannot be created.
   1769      * @throws IllegalArgumentException if name is not valid.
   1770      * @throws NullPointerException if name is null.
   1771      */
   1772     @NonNull
   1773     public static MediaCodec createByCodecName(@NonNull String name)
   1774             throws IOException {
   1775         return new MediaCodec(
   1776                 name, false /* nameIsType */, false /* unused */);
   1777     }
   1778 
   1779     private MediaCodec(
   1780             @NonNull String name, boolean nameIsType, boolean encoder) {
   1781         Looper looper;
   1782         if ((looper = Looper.myLooper()) != null) {
   1783             mEventHandler = new EventHandler(this, looper);
   1784         } else if ((looper = Looper.getMainLooper()) != null) {
   1785             mEventHandler = new EventHandler(this, looper);
   1786         } else {
   1787             mEventHandler = null;
   1788         }
   1789         mCallbackHandler = mEventHandler;
   1790         mOnFrameRenderedHandler = mEventHandler;
   1791 
   1792         mBufferLock = new Object();
   1793 
   1794         native_setup(name, nameIsType, encoder);
   1795     }
   1796 
   1797     @Override
   1798     protected void finalize() {
   1799         native_finalize();
   1800     }
   1801 
   1802     /**
   1803      * Returns the codec to its initial (Uninitialized) state.
   1804      *
   1805      * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable}
   1806      * error has occured to reset the codec to its initial state after creation.
   1807      *
   1808      * @throws CodecException if an unrecoverable error has occured and the codec
   1809      * could not be reset.
   1810      * @throws IllegalStateException if in the Released state.
   1811      */
   1812     public final void reset() {
   1813         freeAllTrackedBuffers(); // free buffers first
   1814         native_reset();
   1815     }
   1816 
   1817     private native final void native_reset();
   1818 
   1819     /**
   1820      * Free up resources used by the codec instance.
   1821      *
   1822      * Make sure you call this when you're done to free up any opened
   1823      * component instance instead of relying on the garbage collector
   1824      * to do this for you at some point in the future.
   1825      */
   1826     public final void release() {
   1827         freeAllTrackedBuffers(); // free buffers first
   1828         native_release();
   1829     }
   1830 
   1831     private native final void native_release();
   1832 
   1833     /**
   1834      * If this codec is to be used as an encoder, pass this flag.
   1835      */
   1836     public static final int CONFIGURE_FLAG_ENCODE = 1;
   1837 
   1838     /** @hide */
   1839     @IntDef(flag = true, value = { CONFIGURE_FLAG_ENCODE })
   1840     @Retention(RetentionPolicy.SOURCE)
   1841     public @interface ConfigureFlag {}
   1842 
   1843     /**
   1844      * Configures a component.
   1845      *
   1846      * @param format The format of the input data (decoder) or the desired
   1847      *               format of the output data (encoder). Passing {@code null}
   1848      *               as {@code format} is equivalent to passing an
   1849      *               {@link MediaFormat#MediaFormat an empty mediaformat}.
   1850      * @param surface Specify a surface on which to render the output of this
   1851      *                decoder. Pass {@code null} as {@code surface} if the
   1852      *                codec does not generate raw video output (e.g. not a video
   1853      *                decoder) and/or if you want to configure the codec for
   1854      *                {@link ByteBuffer} output.
   1855      * @param crypto  Specify a crypto object to facilitate secure decryption
   1856      *                of the media data. Pass {@code null} as {@code crypto} for
   1857      *                non-secure codecs.
   1858      * @param flags   Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the
   1859      *                component as an encoder.
   1860      * @throws IllegalArgumentException if the surface has been released (or is invalid),
   1861      * or the format is unacceptable (e.g. missing a mandatory key),
   1862      * or the flags are not set properly
   1863      * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder).
   1864      * @throws IllegalStateException if not in the Uninitialized state.
   1865      * @throws CryptoException upon DRM error.
   1866      * @throws CodecException upon codec error.
   1867      */
   1868     public void configure(
   1869             @Nullable MediaFormat format,
   1870             @Nullable Surface surface, @Nullable MediaCrypto crypto,
   1871             @ConfigureFlag int flags) {
   1872         configure(format, surface, crypto, null, flags);
   1873     }
   1874 
   1875     /**
   1876      * Configure a component to be used with a descrambler.
   1877      * @param format The format of the input data (decoder) or the desired
   1878      *               format of the output data (encoder). Passing {@code null}
   1879      *               as {@code format} is equivalent to passing an
   1880      *               {@link MediaFormat#MediaFormat an empty mediaformat}.
   1881      * @param surface Specify a surface on which to render the output of this
   1882      *                decoder. Pass {@code null} as {@code surface} if the
   1883      *                codec does not generate raw video output (e.g. not a video
   1884      *                decoder) and/or if you want to configure the codec for
   1885      *                {@link ByteBuffer} output.
   1886      * @param flags   Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the
   1887      *                component as an encoder.
   1888      * @param descrambler Specify a descrambler object to facilitate secure
   1889      *                descrambling of the media data, or null for non-secure codecs.
   1890      * @throws IllegalArgumentException if the surface has been released (or is invalid),
   1891      * or the format is unacceptable (e.g. missing a mandatory key),
   1892      * or the flags are not set properly
   1893      * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder).
   1894      * @throws IllegalStateException if not in the Uninitialized state.
   1895      * @throws CryptoException upon DRM error.
   1896      * @throws CodecException upon codec error.
   1897      */
   1898     public void configure(
   1899             @Nullable MediaFormat format, @Nullable Surface surface,
   1900             @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) {
   1901         configure(format, surface, null,
   1902                 descrambler != null ? descrambler.getBinder() : null, flags);
   1903     }
   1904 
   1905     private void configure(
   1906             @Nullable MediaFormat format, @Nullable Surface surface,
   1907             @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder,
   1908             @ConfigureFlag int flags) {
   1909         if (crypto != null && descramblerBinder != null) {
   1910             throw new IllegalArgumentException("Can't use crypto and descrambler together!");
   1911         }
   1912 
   1913         String[] keys = null;
   1914         Object[] values = null;
   1915 
   1916         if (format != null) {
   1917             Map<String, Object> formatMap = format.getMap();
   1918             keys = new String[formatMap.size()];
   1919             values = new Object[formatMap.size()];
   1920 
   1921             int i = 0;
   1922             for (Map.Entry<String, Object> entry: formatMap.entrySet()) {
   1923                 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) {
   1924                     int sessionId = 0;
   1925                     try {
   1926                         sessionId = (Integer)entry.getValue();
   1927                     }
   1928                     catch (Exception e) {
   1929                         throw new IllegalArgumentException("Wrong Session ID Parameter!");
   1930                     }
   1931                     keys[i] = "audio-hw-sync";
   1932                     values[i] = AudioSystem.getAudioHwSyncForSession(sessionId);
   1933                 } else {
   1934                     keys[i] = entry.getKey();
   1935                     values[i] = entry.getValue();
   1936                 }
   1937                 ++i;
   1938             }
   1939         }
   1940 
   1941         mHasSurface = surface != null;
   1942 
   1943         native_configure(keys, values, surface, crypto, descramblerBinder, flags);
   1944     }
   1945 
   1946     /**
   1947      *  Dynamically sets the output surface of a codec.
   1948      *  <p>
   1949      *  This can only be used if the codec was configured with an output surface.  The
   1950      *  new output surface should have a compatible usage type to the original output surface.
   1951      *  E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output
   1952      *  to ImageReader (software readable) output.
   1953      *  @param surface the output surface to use. It must not be {@code null}.
   1954      *  @throws IllegalStateException if the codec does not support setting the output
   1955      *            surface in the current state.
   1956      *  @throws IllegalArgumentException if the new surface is not of a suitable type for the codec.
   1957      */
   1958     public void setOutputSurface(@NonNull Surface surface) {
   1959         if (!mHasSurface) {
   1960             throw new IllegalStateException("codec was not configured for an output surface");
   1961         }
   1962         native_setSurface(surface);
   1963     }
   1964 
   1965     private native void native_setSurface(@NonNull Surface surface);
   1966 
   1967     /**
   1968      * Create a persistent input surface that can be used with codecs that normally have an input
   1969      * surface, such as video encoders. A persistent input can be reused by subsequent
   1970      * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at
   1971      * most one codec or recorder instance concurrently.
   1972      * <p>
   1973      * The application is responsible for calling release() on the Surface when done.
   1974      *
   1975      * @return an input surface that can be used with {@link #setInputSurface}.
   1976      */
   1977     @NonNull
   1978     public static Surface createPersistentInputSurface() {
   1979         return native_createPersistentInputSurface();
   1980     }
   1981 
   1982     static class PersistentSurface extends Surface {
   1983         @SuppressWarnings("unused")
   1984         PersistentSurface() {} // used by native
   1985 
   1986         @Override
   1987         public void release() {
   1988             native_releasePersistentInputSurface(this);
   1989             super.release();
   1990         }
   1991 
   1992         private long mPersistentObject;
   1993     };
   1994 
   1995     /**
   1996      * Configures the codec (e.g. encoder) to use a persistent input surface in place of input
   1997      * buffers.  This may only be called after {@link #configure} and before {@link #start}, in
   1998      * lieu of {@link #createInputSurface}.
   1999      * @param surface a persistent input surface created by {@link #createPersistentInputSurface}
   2000      * @throws IllegalStateException if not in the Configured state or does not require an input
   2001      *           surface.
   2002      * @throws IllegalArgumentException if the surface was not created by
   2003      *           {@link #createPersistentInputSurface}.
   2004      */
   2005     public void setInputSurface(@NonNull Surface surface) {
   2006         if (!(surface instanceof PersistentSurface)) {
   2007             throw new IllegalArgumentException("not a PersistentSurface");
   2008         }
   2009         native_setInputSurface(surface);
   2010     }
   2011 
   2012     @NonNull
   2013     private static native final PersistentSurface native_createPersistentInputSurface();
   2014     private static native final void native_releasePersistentInputSurface(@NonNull Surface surface);
   2015     private native final void native_setInputSurface(@NonNull Surface surface);
   2016 
   2017     private native final void native_setCallback(@Nullable Callback cb);
   2018 
   2019     private native final void native_configure(
   2020             @Nullable String[] keys, @Nullable Object[] values,
   2021             @Nullable Surface surface, @Nullable MediaCrypto crypto,
   2022             @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags);
   2023 
   2024     /**
   2025      * Requests a Surface to use as the input to an encoder, in place of input buffers.  This
   2026      * may only be called after {@link #configure} and before {@link #start}.
   2027      * <p>
   2028      * The application is responsible for calling release() on the Surface when
   2029      * done.
   2030      * <p>
   2031      * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES.
   2032      * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce
   2033      * unexpected results.
   2034      * @throws IllegalStateException if not in the Configured state.
   2035      */
   2036     @NonNull
   2037     public native final Surface createInputSurface();
   2038 
   2039     /**
   2040      * After successfully configuring the component, call {@code start}.
   2041      * <p>
   2042      * Call {@code start} also if the codec is configured in asynchronous mode,
   2043      * and it has just been flushed, to resume requesting input buffers.
   2044      * @throws IllegalStateException if not in the Configured state
   2045      *         or just after {@link #flush} for a codec that is configured
   2046      *         in asynchronous mode.
   2047      * @throws MediaCodec.CodecException upon codec error. Note that some codec errors
   2048      * for start may be attributed to future method calls.
   2049      */
   2050     public final void start() {
   2051         native_start();
   2052         synchronized(mBufferLock) {
   2053             cacheBuffers(true /* input */);
   2054             cacheBuffers(false /* input */);
   2055         }
   2056     }
   2057     private native final void native_start();
   2058 
   2059     /**
   2060      * Finish the decode/encode session, note that the codec instance
   2061      * remains active and ready to be {@link #start}ed again.
   2062      * To ensure that it is available to other client call {@link #release}
   2063      * and don't just rely on garbage collection to eventually do this for you.
   2064      * @throws IllegalStateException if in the Released state.
   2065      */
   2066     public final void stop() {
   2067         native_stop();
   2068         freeAllTrackedBuffers();
   2069 
   2070         synchronized (mListenerLock) {
   2071             if (mCallbackHandler != null) {
   2072                 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK);
   2073                 mCallbackHandler.removeMessages(EVENT_CALLBACK);
   2074             }
   2075             if (mOnFrameRenderedHandler != null) {
   2076                 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED);
   2077             }
   2078         }
   2079     }
   2080 
   2081     private native final void native_stop();
   2082 
   2083     /**
   2084      * Flush both input and output ports of the component.
   2085      * <p>
   2086      * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer
   2087      * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} &mdash; or obtained
   2088      * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or
   2089      * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks &mdash; become
   2090      * invalid, and all buffers are owned by the codec.
   2091      * <p>
   2092      * If the codec is configured in asynchronous mode, call {@link #start}
   2093      * after {@code flush} has returned to resume codec operations. The codec
   2094      * will not request input buffers until this has happened.
   2095      * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable}
   2096      * callbacks that were not handled prior to calling {@code flush}.
   2097      * The indices returned via these callbacks also become invalid upon calling {@code flush} and
   2098      * should be discarded.</strong>
   2099      * <p>
   2100      * If the codec is configured in synchronous mode, codec will resume
   2101      * automatically if it is configured with an input surface.  Otherwise, it
   2102      * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called.
   2103      *
   2104      * @throws IllegalStateException if not in the Executing state.
   2105      * @throws MediaCodec.CodecException upon codec error.
   2106      */
   2107     public final void flush() {
   2108         synchronized(mBufferLock) {
   2109             invalidateByteBuffers(mCachedInputBuffers);
   2110             invalidateByteBuffers(mCachedOutputBuffers);
   2111             mDequeuedInputBuffers.clear();
   2112             mDequeuedOutputBuffers.clear();
   2113         }
   2114         native_flush();
   2115     }
   2116 
   2117     private native final void native_flush();
   2118 
   2119     /**
   2120      * Thrown when an internal codec error occurs.
   2121      */
   2122     public final static class CodecException extends IllegalStateException {
   2123         CodecException(int errorCode, int actionCode, @Nullable String detailMessage) {
   2124             super(detailMessage);
   2125             mErrorCode = errorCode;
   2126             mActionCode = actionCode;
   2127 
   2128             // TODO get this from codec
   2129             final String sign = errorCode < 0 ? "neg_" : "";
   2130             mDiagnosticInfo =
   2131                 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode);
   2132         }
   2133 
   2134         /**
   2135          * Returns true if the codec exception is a transient issue,
   2136          * perhaps due to resource constraints, and that the method
   2137          * (or encoding/decoding) may be retried at a later time.
   2138          */
   2139         public boolean isTransient() {
   2140             return mActionCode == ACTION_TRANSIENT;
   2141         }
   2142 
   2143         /**
   2144          * Returns true if the codec cannot proceed further,
   2145          * but can be recovered by stopping, configuring,
   2146          * and starting again.
   2147          */
   2148         public boolean isRecoverable() {
   2149             return mActionCode == ACTION_RECOVERABLE;
   2150         }
   2151 
   2152         /**
   2153          * Retrieve the error code associated with a CodecException
   2154          */
   2155         public int getErrorCode() {
   2156             return mErrorCode;
   2157         }
   2158 
   2159         /**
   2160          * Retrieve a developer-readable diagnostic information string
   2161          * associated with the exception. Do not show this to end-users,
   2162          * since this string will not be localized or generally
   2163          * comprehensible to end-users.
   2164          */
   2165         public @NonNull String getDiagnosticInfo() {
   2166             return mDiagnosticInfo;
   2167         }
   2168 
   2169         /**
   2170          * This indicates required resource was not able to be allocated.
   2171          */
   2172         public static final int ERROR_INSUFFICIENT_RESOURCE = 1100;
   2173 
   2174         /**
   2175          * This indicates the resource manager reclaimed the media resource used by the codec.
   2176          * <p>
   2177          * With this exception, the codec must be released, as it has moved to terminal state.
   2178          */
   2179         public static final int ERROR_RECLAIMED = 1101;
   2180 
   2181         /** @hide */
   2182         @IntDef({
   2183             ERROR_INSUFFICIENT_RESOURCE,
   2184             ERROR_RECLAIMED,
   2185         })
   2186         @Retention(RetentionPolicy.SOURCE)
   2187         public @interface ReasonCode {}
   2188 
   2189         /* Must be in sync with android_media_MediaCodec.cpp */
   2190         private final static int ACTION_TRANSIENT = 1;
   2191         private final static int ACTION_RECOVERABLE = 2;
   2192 
   2193         private final String mDiagnosticInfo;
   2194         private final int mErrorCode;
   2195         private final int mActionCode;
   2196     }
   2197 
   2198     /**
   2199      * Thrown when a crypto error occurs while queueing a secure input buffer.
   2200      */
   2201     public final static class CryptoException extends RuntimeException {
   2202         public CryptoException(int errorCode, @Nullable String detailMessage) {
   2203             super(detailMessage);
   2204             mErrorCode = errorCode;
   2205         }
   2206 
   2207         /**
   2208          * This indicates that the requested key was not found when trying to
   2209          * perform a decrypt operation.  The operation can be retried after adding
   2210          * the correct decryption key.
   2211          */
   2212         public static final int ERROR_NO_KEY = 1;
   2213 
   2214         /**
   2215          * This indicates that the key used for decryption is no longer
   2216          * valid due to license term expiration.  The operation can be retried
   2217          * after updating the expired keys.
   2218          */
   2219         public static final int ERROR_KEY_EXPIRED = 2;
   2220 
   2221         /**
   2222          * This indicates that a required crypto resource was not able to be
   2223          * allocated while attempting the requested operation.  The operation
   2224          * can be retried if the app is able to release resources.
   2225          */
   2226         public static final int ERROR_RESOURCE_BUSY = 3;
   2227 
   2228         /**
   2229          * This indicates that the output protection levels supported by the
   2230          * device are not sufficient to meet the requirements set by the
   2231          * content owner in the license policy.
   2232          */
   2233         public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 4;
   2234 
   2235         /**
   2236          * This indicates that decryption was attempted on a session that is
   2237          * not opened, which could be due to a failure to open the session,
   2238          * closing the session prematurely, or the session being reclaimed
   2239          * by the resource manager.
   2240          */
   2241         public static final int ERROR_SESSION_NOT_OPENED = 5;
   2242 
   2243         /**
   2244          * This indicates that an operation was attempted that could not be
   2245          * supported by the crypto system of the device in its current
   2246          * configuration.  It may occur when the license policy requires
   2247          * device security features that aren't supported by the device,
   2248          * or due to an internal error in the crypto system that prevents
   2249          * the specified security policy from being met.
   2250          */
   2251         public static final int ERROR_UNSUPPORTED_OPERATION = 6;
   2252 
   2253         /** @hide */
   2254         @IntDef({
   2255             ERROR_NO_KEY,
   2256             ERROR_KEY_EXPIRED,
   2257             ERROR_RESOURCE_BUSY,
   2258             ERROR_INSUFFICIENT_OUTPUT_PROTECTION,
   2259             ERROR_SESSION_NOT_OPENED,
   2260             ERROR_UNSUPPORTED_OPERATION
   2261         })
   2262         @Retention(RetentionPolicy.SOURCE)
   2263         public @interface CryptoErrorCode {}
   2264 
   2265         /**
   2266          * Retrieve the error code associated with a CryptoException
   2267          */
   2268         @CryptoErrorCode
   2269         public int getErrorCode() {
   2270             return mErrorCode;
   2271         }
   2272 
   2273         private int mErrorCode;
   2274     }
   2275 
   2276     /**
   2277      * After filling a range of the input buffer at the specified index
   2278      * submit it to the component. Once an input buffer is queued to
   2279      * the codec, it MUST NOT be used until it is later retrieved by
   2280      * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer}
   2281      * return value or a {@link Callback#onInputBufferAvailable}
   2282      * callback.
   2283      * <p>
   2284      * Many decoders require the actual compressed data stream to be
   2285      * preceded by "codec specific data", i.e. setup data used to initialize
   2286      * the codec such as PPS/SPS in the case of AVC video or code tables
   2287      * in the case of vorbis audio.
   2288      * The class {@link android.media.MediaExtractor} provides codec
   2289      * specific data as part of
   2290      * the returned track format in entries named "csd-0", "csd-1" ...
   2291      * <p>
   2292      * These buffers can be submitted directly after {@link #start} or
   2293      * {@link #flush} by specifying the flag {@link
   2294      * #BUFFER_FLAG_CODEC_CONFIG}.  However, if you configure the
   2295      * codec with a {@link MediaFormat} containing these keys, they
   2296      * will be automatically submitted by MediaCodec directly after
   2297      * start.  Therefore, the use of {@link
   2298      * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is
   2299      * recommended only for advanced users.
   2300      * <p>
   2301      * To indicate that this is the final piece of input data (or rather that
   2302      * no more input data follows unless the decoder is subsequently flushed)
   2303      * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}.
   2304      * <p class=note>
   2305      * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M},
   2306      * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered)
   2307      * Surface output buffers, and the resulting frame timestamp was undefined.
   2308      * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set.
   2309      * Similarly, since frame timestamps can be used by the destination surface for rendering
   2310      * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be
   2311      * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long)
   2312      * SurfaceView specifics}).</strong>
   2313      *
   2314      * @param index The index of a client-owned input buffer previously returned
   2315      *              in a call to {@link #dequeueInputBuffer}.
   2316      * @param offset The byte offset into the input buffer at which the data starts.
   2317      * @param size The number of bytes of valid input data.
   2318      * @param presentationTimeUs The presentation timestamp in microseconds for this
   2319      *                           buffer. This is normally the media time at which this
   2320      *                           buffer should be presented (rendered). When using an output
   2321      *                           surface, this will be propagated as the {@link
   2322      *                           SurfaceTexture#getTimestamp timestamp} for the frame (after
   2323      *                           conversion to nanoseconds).
   2324      * @param flags A bitmask of flags
   2325      *              {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}.
   2326      *              While not prohibited, most codecs do not use the
   2327      *              {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers.
   2328      * @throws IllegalStateException if not in the Executing state.
   2329      * @throws MediaCodec.CodecException upon codec error.
   2330      * @throws CryptoException if a crypto object has been specified in
   2331      *         {@link #configure}
   2332      */
   2333     public final void queueInputBuffer(
   2334             int index,
   2335             int offset, int size, long presentationTimeUs, int flags)
   2336         throws CryptoException {
   2337         synchronized(mBufferLock) {
   2338             invalidateByteBuffer(mCachedInputBuffers, index);
   2339             mDequeuedInputBuffers.remove(index);
   2340         }
   2341         try {
   2342             native_queueInputBuffer(
   2343                     index, offset, size, presentationTimeUs, flags);
   2344         } catch (CryptoException | IllegalStateException e) {
   2345             revalidateByteBuffer(mCachedInputBuffers, index);
   2346             throw e;
   2347         }
   2348     }
   2349 
   2350     private native final void native_queueInputBuffer(
   2351             int index,
   2352             int offset, int size, long presentationTimeUs, int flags)
   2353         throws CryptoException;
   2354 
   2355     public static final int CRYPTO_MODE_UNENCRYPTED = 0;
   2356     public static final int CRYPTO_MODE_AES_CTR     = 1;
   2357     public static final int CRYPTO_MODE_AES_CBC     = 2;
   2358 
   2359     /**
   2360      * Metadata describing the structure of a (at least partially) encrypted
   2361      * input sample.
   2362      * A buffer's data is considered to be partitioned into "subSamples",
   2363      * each subSample starts with a (potentially empty) run of plain,
   2364      * unencrypted bytes followed by a (also potentially empty) run of
   2365      * encrypted bytes. If pattern encryption applies, each of the latter runs
   2366      * is encrypted only partly, according to a repeating pattern of "encrypt"
   2367      * and "skip" blocks. numBytesOfClearData can be null to indicate that all
   2368      * data is encrypted. This information encapsulates per-sample metadata as
   2369      * outlined in ISO/IEC FDIS 23001-7:2011 "Common encryption in ISO base
   2370      * media file format files".
   2371      */
   2372     public final static class CryptoInfo {
   2373         /**
   2374          * The number of subSamples that make up the buffer's contents.
   2375          */
   2376         public int numSubSamples;
   2377         /**
   2378          * The number of leading unencrypted bytes in each subSample.
   2379          */
   2380         public int[] numBytesOfClearData;
   2381         /**
   2382          * The number of trailing encrypted bytes in each subSample.
   2383          */
   2384         public int[] numBytesOfEncryptedData;
   2385         /**
   2386          * A 16-byte key id
   2387          */
   2388         public byte[] key;
   2389         /**
   2390          * A 16-byte initialization vector
   2391          */
   2392         public byte[] iv;
   2393         /**
   2394          * The type of encryption that has been applied,
   2395          * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR}
   2396          * and {@link #CRYPTO_MODE_AES_CBC}
   2397          */
   2398         public int mode;
   2399 
   2400         /**
   2401          * Metadata describing an encryption pattern for the protected bytes in
   2402          * a subsample.  An encryption pattern consists of a repeating sequence
   2403          * of crypto blocks comprised of a number of encrypted blocks followed
   2404          * by a number of unencrypted, or skipped, blocks.
   2405          */
   2406         public final static class Pattern {
   2407             /**
   2408              * Number of blocks to be encrypted in the pattern. If zero, pattern
   2409              * encryption is inoperative.
   2410              */
   2411             private int mEncryptBlocks;
   2412 
   2413             /**
   2414              * Number of blocks to be skipped (left clear) in the pattern. If zero,
   2415              * pattern encryption is inoperative.
   2416              */
   2417             private int mSkipBlocks;
   2418 
   2419             /**
   2420              * Construct a sample encryption pattern given the number of blocks to
   2421              * encrypt and skip in the pattern.
   2422              */
   2423             public Pattern(int blocksToEncrypt, int blocksToSkip) {
   2424                 set(blocksToEncrypt, blocksToSkip);
   2425             }
   2426 
   2427             /**
   2428              * Set the number of blocks to encrypt and skip in a sample encryption
   2429              * pattern.
   2430              */
   2431             public void set(int blocksToEncrypt, int blocksToSkip) {
   2432                 mEncryptBlocks = blocksToEncrypt;
   2433                 mSkipBlocks = blocksToSkip;
   2434             }
   2435 
   2436             /**
   2437              * Return the number of blocks to skip in a sample encryption pattern.
   2438              */
   2439             public int getSkipBlocks() {
   2440                 return mSkipBlocks;
   2441             }
   2442 
   2443             /**
   2444              * Return the number of blocks to encrypt in a sample encryption pattern.
   2445              */
   2446             public int getEncryptBlocks() {
   2447                 return mEncryptBlocks;
   2448             }
   2449         };
   2450 
   2451         private final Pattern zeroPattern = new Pattern(0, 0);
   2452 
   2453         /**
   2454          * The pattern applicable to the protected data in each subsample.
   2455          */
   2456         private Pattern pattern;
   2457 
   2458         /**
   2459          * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of
   2460          * a {@link MediaCodec.CryptoInfo} instance.
   2461          */
   2462         public void set(
   2463                 int newNumSubSamples,
   2464                 @NonNull int[] newNumBytesOfClearData,
   2465                 @NonNull int[] newNumBytesOfEncryptedData,
   2466                 @NonNull byte[] newKey,
   2467                 @NonNull byte[] newIV,
   2468                 int newMode) {
   2469             numSubSamples = newNumSubSamples;
   2470             numBytesOfClearData = newNumBytesOfClearData;
   2471             numBytesOfEncryptedData = newNumBytesOfEncryptedData;
   2472             key = newKey;
   2473             iv = newIV;
   2474             mode = newMode;
   2475             pattern = zeroPattern;
   2476         }
   2477 
   2478         /**
   2479          * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance.
   2480          * See {@link MediaCodec.CryptoInfo.Pattern}.
   2481          */
   2482         public void setPattern(Pattern newPattern) {
   2483             pattern = newPattern;
   2484         }
   2485 
   2486         @Override
   2487         public String toString() {
   2488             StringBuilder builder = new StringBuilder();
   2489             builder.append(numSubSamples + " subsamples, key [");
   2490             String hexdigits = "0123456789abcdef";
   2491             for (int i = 0; i < key.length; i++) {
   2492                 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4));
   2493                 builder.append(hexdigits.charAt(key[i] & 0x0f));
   2494             }
   2495             builder.append("], iv [");
   2496             for (int i = 0; i < key.length; i++) {
   2497                 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4));
   2498                 builder.append(hexdigits.charAt(iv[i] & 0x0f));
   2499             }
   2500             builder.append("], clear ");
   2501             builder.append(Arrays.toString(numBytesOfClearData));
   2502             builder.append(", encrypted ");
   2503             builder.append(Arrays.toString(numBytesOfEncryptedData));
   2504             return builder.toString();
   2505         }
   2506     };
   2507 
   2508     /**
   2509      * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is
   2510      * potentially encrypted.
   2511      * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong>
   2512      *
   2513      * @param index The index of a client-owned input buffer previously returned
   2514      *              in a call to {@link #dequeueInputBuffer}.
   2515      * @param offset The byte offset into the input buffer at which the data starts.
   2516      * @param info Metadata required to facilitate decryption, the object can be
   2517      *             reused immediately after this call returns.
   2518      * @param presentationTimeUs The presentation timestamp in microseconds for this
   2519      *                           buffer. This is normally the media time at which this
   2520      *                           buffer should be presented (rendered).
   2521      * @param flags A bitmask of flags
   2522      *              {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}.
   2523      *              While not prohibited, most codecs do not use the
   2524      *              {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers.
   2525      * @throws IllegalStateException if not in the Executing state.
   2526      * @throws MediaCodec.CodecException upon codec error.
   2527      * @throws CryptoException if an error occurs while attempting to decrypt the buffer.
   2528      *              An error code associated with the exception helps identify the
   2529      *              reason for the failure.
   2530      */
   2531     public final void queueSecureInputBuffer(
   2532             int index,
   2533             int offset,
   2534             @NonNull CryptoInfo info,
   2535             long presentationTimeUs,
   2536             int flags) throws CryptoException {
   2537         synchronized(mBufferLock) {
   2538             invalidateByteBuffer(mCachedInputBuffers, index);
   2539             mDequeuedInputBuffers.remove(index);
   2540         }
   2541         try {
   2542             native_queueSecureInputBuffer(
   2543                     index, offset, info, presentationTimeUs, flags);
   2544         } catch (CryptoException | IllegalStateException e) {
   2545             revalidateByteBuffer(mCachedInputBuffers, index);
   2546             throw e;
   2547         }
   2548     }
   2549 
   2550     private native final void native_queueSecureInputBuffer(
   2551             int index,
   2552             int offset,
   2553             @NonNull CryptoInfo info,
   2554             long presentationTimeUs,
   2555             int flags) throws CryptoException;
   2556 
   2557     /**
   2558      * Returns the index of an input buffer to be filled with valid data
   2559      * or -1 if no such buffer is currently available.
   2560      * This method will return immediately if timeoutUs == 0, wait indefinitely
   2561      * for the availability of an input buffer if timeoutUs &lt; 0 or wait up
   2562      * to "timeoutUs" microseconds if timeoutUs &gt; 0.
   2563      * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite".
   2564      * @throws IllegalStateException if not in the Executing state,
   2565      *         or codec is configured in asynchronous mode.
   2566      * @throws MediaCodec.CodecException upon codec error.
   2567      */
   2568     public final int dequeueInputBuffer(long timeoutUs) {
   2569         int res = native_dequeueInputBuffer(timeoutUs);
   2570         if (res >= 0) {
   2571             synchronized(mBufferLock) {
   2572                 validateInputByteBuffer(mCachedInputBuffers, res);
   2573             }
   2574         }
   2575         return res;
   2576     }
   2577 
   2578     private native final int native_dequeueInputBuffer(long timeoutUs);
   2579 
   2580     /**
   2581      * If a non-negative timeout had been specified in the call
   2582      * to {@link #dequeueOutputBuffer}, indicates that the call timed out.
   2583      */
   2584     public static final int INFO_TRY_AGAIN_LATER        = -1;
   2585 
   2586     /**
   2587      * The output format has changed, subsequent data will follow the new
   2588      * format. {@link #getOutputFormat()} returns the new format.  Note, that
   2589      * you can also use the new {@link #getOutputFormat(int)} method to
   2590      * get the format for a specific output buffer.  This frees you from
   2591      * having to track output format changes.
   2592      */
   2593     public static final int INFO_OUTPUT_FORMAT_CHANGED  = -2;
   2594 
   2595     /**
   2596      * The output buffers have changed, the client must refer to the new
   2597      * set of output buffers returned by {@link #getOutputBuffers} from
   2598      * this point on.
   2599      *
   2600      * <p>Additionally, this event signals that the video scaling mode
   2601      * may have been reset to the default.</p>
   2602      *
   2603      * @deprecated This return value can be ignored as {@link
   2604      * #getOutputBuffers} has been deprecated.  Client should
   2605      * request a current buffer using on of the get-buffer or
   2606      * get-image methods each time one has been dequeued.
   2607      */
   2608     public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3;
   2609 
   2610     /** @hide */
   2611     @IntDef({
   2612         INFO_TRY_AGAIN_LATER,
   2613         INFO_OUTPUT_FORMAT_CHANGED,
   2614         INFO_OUTPUT_BUFFERS_CHANGED,
   2615     })
   2616     @Retention(RetentionPolicy.SOURCE)
   2617     public @interface OutputBufferInfo {}
   2618 
   2619     /**
   2620      * Dequeue an output buffer, block at most "timeoutUs" microseconds.
   2621      * Returns the index of an output buffer that has been successfully
   2622      * decoded or one of the INFO_* constants.
   2623      * @param info Will be filled with buffer meta data.
   2624      * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite".
   2625      * @throws IllegalStateException if not in the Executing state,
   2626      *         or codec is configured in asynchronous mode.
   2627      * @throws MediaCodec.CodecException upon codec error.
   2628      */
   2629     @OutputBufferInfo
   2630     public final int dequeueOutputBuffer(
   2631             @NonNull BufferInfo info, long timeoutUs) {
   2632         int res = native_dequeueOutputBuffer(info, timeoutUs);
   2633         synchronized(mBufferLock) {
   2634             if (res == INFO_OUTPUT_BUFFERS_CHANGED) {
   2635                 cacheBuffers(false /* input */);
   2636             } else if (res >= 0) {
   2637                 validateOutputByteBuffer(mCachedOutputBuffers, res, info);
   2638                 if (mHasSurface) {
   2639                     mDequeuedOutputInfos.put(res, info.dup());
   2640                 }
   2641             }
   2642         }
   2643         return res;
   2644     }
   2645 
   2646     private native final int native_dequeueOutputBuffer(
   2647             @NonNull BufferInfo info, long timeoutUs);
   2648 
   2649     /**
   2650      * If you are done with a buffer, use this call to return the buffer to the codec
   2651      * or to render it on the output surface. If you configured the codec with an
   2652      * output surface, setting {@code render} to {@code true} will first send the buffer
   2653      * to that output surface. The surface will release the buffer back to the codec once
   2654      * it is no longer used/displayed.
   2655      *
   2656      * Once an output buffer is released to the codec, it MUST NOT
   2657      * be used until it is later retrieved by {@link #getOutputBuffer} in response
   2658      * to a {@link #dequeueOutputBuffer} return value or a
   2659      * {@link Callback#onOutputBufferAvailable} callback.
   2660      *
   2661      * @param index The index of a client-owned output buffer previously returned
   2662      *              from a call to {@link #dequeueOutputBuffer}.
   2663      * @param render If a valid surface was specified when configuring the codec,
   2664      *               passing true renders this output buffer to the surface.
   2665      * @throws IllegalStateException if not in the Executing state.
   2666      * @throws MediaCodec.CodecException upon codec error.
   2667      */
   2668     public final void releaseOutputBuffer(int index, boolean render) {
   2669         BufferInfo info = null;
   2670         synchronized(mBufferLock) {
   2671             invalidateByteBuffer(mCachedOutputBuffers, index);
   2672             mDequeuedOutputBuffers.remove(index);
   2673             if (mHasSurface) {
   2674                 info = mDequeuedOutputInfos.remove(index);
   2675             }
   2676         }
   2677         releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */);
   2678     }
   2679 
   2680     /**
   2681      * If you are done with a buffer, use this call to update its surface timestamp
   2682      * and return it to the codec to render it on the output surface. If you
   2683      * have not specified an output surface when configuring this video codec,
   2684      * this call will simply return the buffer to the codec.<p>
   2685      *
   2686      * The timestamp may have special meaning depending on the destination surface.
   2687      *
   2688      * <table>
   2689      * <tr><th>SurfaceView specifics</th></tr>
   2690      * <tr><td>
   2691      * If you render your buffer on a {@link android.view.SurfaceView},
   2692      * you can use the timestamp to render the buffer at a specific time (at the
   2693      * VSYNC at or after the buffer timestamp).  For this to work, the timestamp
   2694      * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}.
   2695      * Currently, this is set as within one (1) second. A few notes:
   2696      *
   2697      * <ul>
   2698      * <li>the buffer will not be returned to the codec until the timestamp
   2699      * has passed and the buffer is no longer used by the {@link android.view.Surface}.
   2700      * <li>buffers are processed sequentially, so you may block subsequent buffers to
   2701      * be displayed on the {@link android.view.Surface}.  This is important if you
   2702      * want to react to user action, e.g. stop the video or seek.
   2703      * <li>if multiple buffers are sent to the {@link android.view.Surface} to be
   2704      * rendered at the same VSYNC, the last one will be shown, and the other ones
   2705      * will be dropped.
   2706      * <li>if the timestamp is <em>not</em> "reasonably close" to the current system
   2707      * time, the {@link android.view.Surface} will ignore the timestamp, and
   2708      * display the buffer at the earliest feasible time.  In this mode it will not
   2709      * drop frames.
   2710      * <li>for best performance and quality, call this method when you are about
   2711      * two VSYNCs' time before the desired render time.  For 60Hz displays, this is
   2712      * about 33 msec.
   2713      * </ul>
   2714      * </td></tr>
   2715      * </table>
   2716      *
   2717      * Once an output buffer is released to the codec, it MUST NOT
   2718      * be used until it is later retrieved by {@link #getOutputBuffer} in response
   2719      * to a {@link #dequeueOutputBuffer} return value or a
   2720      * {@link Callback#onOutputBufferAvailable} callback.
   2721      *
   2722      * @param index The index of a client-owned output buffer previously returned
   2723      *              from a call to {@link #dequeueOutputBuffer}.
   2724      * @param renderTimestampNs The timestamp to associate with this buffer when
   2725      *              it is sent to the Surface.
   2726      * @throws IllegalStateException if not in the Executing state.
   2727      * @throws MediaCodec.CodecException upon codec error.
   2728      */
   2729     public final void releaseOutputBuffer(int index, long renderTimestampNs) {
   2730         BufferInfo info = null;
   2731         synchronized(mBufferLock) {
   2732             invalidateByteBuffer(mCachedOutputBuffers, index);
   2733             mDequeuedOutputBuffers.remove(index);
   2734             if (mHasSurface) {
   2735                 info = mDequeuedOutputInfos.remove(index);
   2736             }
   2737         }
   2738         releaseOutputBuffer(
   2739                 index, true /* render */, true /* updatePTS */, renderTimestampNs);
   2740     }
   2741 
   2742     private native final void releaseOutputBuffer(
   2743             int index, boolean render, boolean updatePTS, long timeNs);
   2744 
   2745     /**
   2746      * Signals end-of-stream on input.  Equivalent to submitting an empty buffer with
   2747      * {@link #BUFFER_FLAG_END_OF_STREAM} set.  This may only be used with
   2748      * encoders receiving input from a Surface created by {@link #createInputSurface}.
   2749      * @throws IllegalStateException if not in the Executing state.
   2750      * @throws MediaCodec.CodecException upon codec error.
   2751      */
   2752     public native final void signalEndOfInputStream();
   2753 
   2754     /**
   2755      * Call this after dequeueOutputBuffer signals a format change by returning
   2756      * {@link #INFO_OUTPUT_FORMAT_CHANGED}.
   2757      * You can also call this after {@link #configure} returns
   2758      * successfully to get the output format initially configured
   2759      * for the codec.  Do this to determine what optional
   2760      * configuration parameters were supported by the codec.
   2761      *
   2762      * @throws IllegalStateException if not in the Executing or
   2763      *                               Configured state.
   2764      * @throws MediaCodec.CodecException upon codec error.
   2765      */
   2766     @NonNull
   2767     public final MediaFormat getOutputFormat() {
   2768         return new MediaFormat(getFormatNative(false /* input */));
   2769     }
   2770 
   2771     /**
   2772      * Call this after {@link #configure} returns successfully to
   2773      * get the input format accepted by the codec. Do this to
   2774      * determine what optional configuration parameters were
   2775      * supported by the codec.
   2776      *
   2777      * @throws IllegalStateException if not in the Executing or
   2778      *                               Configured state.
   2779      * @throws MediaCodec.CodecException upon codec error.
   2780      */
   2781     @NonNull
   2782     public final MediaFormat getInputFormat() {
   2783         return new MediaFormat(getFormatNative(true /* input */));
   2784     }
   2785 
   2786     /**
   2787      * Returns the output format for a specific output buffer.
   2788      *
   2789      * @param index The index of a client-owned input buffer previously
   2790      *              returned from a call to {@link #dequeueInputBuffer}.
   2791      *
   2792      * @return the format for the output buffer, or null if the index
   2793      * is not a dequeued output buffer.
   2794      */
   2795     @NonNull
   2796     public final MediaFormat getOutputFormat(int index) {
   2797         return new MediaFormat(getOutputFormatNative(index));
   2798     }
   2799 
   2800     @NonNull
   2801     private native final Map<String, Object> getFormatNative(boolean input);
   2802 
   2803     @NonNull
   2804     private native final Map<String, Object> getOutputFormatNative(int index);
   2805 
   2806     // used to track dequeued buffers
   2807     private static class BufferMap {
   2808         // various returned representations of the codec buffer
   2809         private static class CodecBuffer {
   2810             private Image mImage;
   2811             private ByteBuffer mByteBuffer;
   2812 
   2813             public void free() {
   2814                 if (mByteBuffer != null) {
   2815                     // all of our ByteBuffers are direct
   2816                     java.nio.NioUtils.freeDirectBuffer(mByteBuffer);
   2817                     mByteBuffer = null;
   2818                 }
   2819                 if (mImage != null) {
   2820                     mImage.close();
   2821                     mImage = null;
   2822                 }
   2823             }
   2824 
   2825             public void setImage(@Nullable Image image) {
   2826                 free();
   2827                 mImage = image;
   2828             }
   2829 
   2830             public void setByteBuffer(@Nullable ByteBuffer buffer) {
   2831                 free();
   2832                 mByteBuffer = buffer;
   2833             }
   2834         }
   2835 
   2836         private final Map<Integer, CodecBuffer> mMap =
   2837             new HashMap<Integer, CodecBuffer>();
   2838 
   2839         public void remove(int index) {
   2840             CodecBuffer buffer = mMap.get(index);
   2841             if (buffer != null) {
   2842                 buffer.free();
   2843                 mMap.remove(index);
   2844             }
   2845         }
   2846 
   2847         public void put(int index, @Nullable ByteBuffer newBuffer) {
   2848             CodecBuffer buffer = mMap.get(index);
   2849             if (buffer == null) { // likely
   2850                 buffer = new CodecBuffer();
   2851                 mMap.put(index, buffer);
   2852             }
   2853             buffer.setByteBuffer(newBuffer);
   2854         }
   2855 
   2856         public void put(int index, @Nullable Image newImage) {
   2857             CodecBuffer buffer = mMap.get(index);
   2858             if (buffer == null) { // likely
   2859                 buffer = new CodecBuffer();
   2860                 mMap.put(index, buffer);
   2861             }
   2862             buffer.setImage(newImage);
   2863         }
   2864 
   2865         public void clear() {
   2866             for (CodecBuffer buffer: mMap.values()) {
   2867                 buffer.free();
   2868             }
   2869             mMap.clear();
   2870         }
   2871     }
   2872 
   2873     private ByteBuffer[] mCachedInputBuffers;
   2874     private ByteBuffer[] mCachedOutputBuffers;
   2875     private final BufferMap mDequeuedInputBuffers = new BufferMap();
   2876     private final BufferMap mDequeuedOutputBuffers = new BufferMap();
   2877     private final Map<Integer, BufferInfo> mDequeuedOutputInfos =
   2878         new HashMap<Integer, BufferInfo>();
   2879     final private Object mBufferLock;
   2880 
   2881     private final void invalidateByteBuffer(
   2882             @Nullable ByteBuffer[] buffers, int index) {
   2883         if (buffers != null && index >= 0 && index < buffers.length) {
   2884             ByteBuffer buffer = buffers[index];
   2885             if (buffer != null) {
   2886                 buffer.setAccessible(false);
   2887             }
   2888         }
   2889     }
   2890 
   2891     private final void validateInputByteBuffer(
   2892             @Nullable ByteBuffer[] buffers, int index) {
   2893         if (buffers != null && index >= 0 && index < buffers.length) {
   2894             ByteBuffer buffer = buffers[index];
   2895             if (buffer != null) {
   2896                 buffer.setAccessible(true);
   2897                 buffer.clear();
   2898             }
   2899         }
   2900     }
   2901 
   2902     private final void revalidateByteBuffer(
   2903             @Nullable ByteBuffer[] buffers, int index) {
   2904         synchronized(mBufferLock) {
   2905             if (buffers != null && index >= 0 && index < buffers.length) {
   2906                 ByteBuffer buffer = buffers[index];
   2907                 if (buffer != null) {
   2908                     buffer.setAccessible(true);
   2909                 }
   2910             }
   2911         }
   2912     }
   2913 
   2914     private final void validateOutputByteBuffer(
   2915             @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) {
   2916         if (buffers != null && index >= 0 && index < buffers.length) {
   2917             ByteBuffer buffer = buffers[index];
   2918             if (buffer != null) {
   2919                 buffer.setAccessible(true);
   2920                 buffer.limit(info.offset + info.size).position(info.offset);
   2921             }
   2922         }
   2923     }
   2924 
   2925     private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) {
   2926         if (buffers != null) {
   2927             for (ByteBuffer buffer: buffers) {
   2928                 if (buffer != null) {
   2929                     buffer.setAccessible(false);
   2930                 }
   2931             }
   2932         }
   2933     }
   2934 
   2935     private final void freeByteBuffer(@Nullable ByteBuffer buffer) {
   2936         if (buffer != null /* && buffer.isDirect() */) {
   2937             // all of our ByteBuffers are direct
   2938             java.nio.NioUtils.freeDirectBuffer(buffer);
   2939         }
   2940     }
   2941 
   2942     private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) {
   2943         if (buffers != null) {
   2944             for (ByteBuffer buffer: buffers) {
   2945                 freeByteBuffer(buffer);
   2946             }
   2947         }
   2948     }
   2949 
   2950     private final void freeAllTrackedBuffers() {
   2951         synchronized(mBufferLock) {
   2952             freeByteBuffers(mCachedInputBuffers);
   2953             freeByteBuffers(mCachedOutputBuffers);
   2954             mCachedInputBuffers = null;
   2955             mCachedOutputBuffers = null;
   2956             mDequeuedInputBuffers.clear();
   2957             mDequeuedOutputBuffers.clear();
   2958         }
   2959     }
   2960 
   2961     private final void cacheBuffers(boolean input) {
   2962         ByteBuffer[] buffers = null;
   2963         try {
   2964             buffers = getBuffers(input);
   2965             invalidateByteBuffers(buffers);
   2966         } catch (IllegalStateException e) {
   2967             // we don't get buffers in async mode
   2968         }
   2969         if (input) {
   2970             mCachedInputBuffers = buffers;
   2971         } else {
   2972             mCachedOutputBuffers = buffers;
   2973         }
   2974     }
   2975 
   2976     /**
   2977      * Retrieve the set of input buffers.  Call this after start()
   2978      * returns. After calling this method, any ByteBuffers
   2979      * previously returned by an earlier call to this method MUST no
   2980      * longer be used.
   2981      *
   2982      * @deprecated Use the new {@link #getInputBuffer} method instead
   2983      * each time an input buffer is dequeued.
   2984      *
   2985      * <b>Note:</b> As of API 21, dequeued input buffers are
   2986      * automatically {@link java.nio.Buffer#clear cleared}.
   2987      *
   2988      * <em>Do not use this method if using an input surface.</em>
   2989      *
   2990      * @throws IllegalStateException if not in the Executing state,
   2991      *         or codec is configured in asynchronous mode.
   2992      * @throws MediaCodec.CodecException upon codec error.
   2993      */
   2994     @NonNull
   2995     public ByteBuffer[] getInputBuffers() {
   2996         if (mCachedInputBuffers == null) {
   2997             throw new IllegalStateException();
   2998         }
   2999         // FIXME: check codec status
   3000         return mCachedInputBuffers;
   3001     }
   3002 
   3003     /**
   3004      * Retrieve the set of output buffers.  Call this after start()
   3005      * returns and whenever dequeueOutputBuffer signals an output
   3006      * buffer change by returning {@link
   3007      * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any
   3008      * ByteBuffers previously returned by an earlier call to this
   3009      * method MUST no longer be used.
   3010      *
   3011      * @deprecated Use the new {@link #getOutputBuffer} method instead
   3012      * each time an output buffer is dequeued.  This method is not
   3013      * supported if codec is configured in asynchronous mode.
   3014      *
   3015      * <b>Note:</b> As of API 21, the position and limit of output
   3016      * buffers that are dequeued will be set to the valid data
   3017      * range.
   3018      *
   3019      * <em>Do not use this method if using an output surface.</em>
   3020      *
   3021      * @throws IllegalStateException if not in the Executing state,
   3022      *         or codec is configured in asynchronous mode.
   3023      * @throws MediaCodec.CodecException upon codec error.
   3024      */
   3025     @NonNull
   3026     public ByteBuffer[] getOutputBuffers() {
   3027         if (mCachedOutputBuffers == null) {
   3028             throw new IllegalStateException();
   3029         }
   3030         // FIXME: check codec status
   3031         return mCachedOutputBuffers;
   3032     }
   3033 
   3034     /**
   3035      * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer
   3036      * object for a dequeued input buffer index to contain the input data.
   3037      *
   3038      * After calling this method any ByteBuffer or Image object
   3039      * previously returned for the same input index MUST no longer
   3040      * be used.
   3041      *
   3042      * @param index The index of a client-owned input buffer previously
   3043      *              returned from a call to {@link #dequeueInputBuffer},
   3044      *              or received via an onInputBufferAvailable callback.
   3045      *
   3046      * @return the input buffer, or null if the index is not a dequeued
   3047      * input buffer, or if the codec is configured for surface input.
   3048      *
   3049      * @throws IllegalStateException if not in the Executing state.
   3050      * @throws MediaCodec.CodecException upon codec error.
   3051      */
   3052     @Nullable
   3053     public ByteBuffer getInputBuffer(int index) {
   3054         ByteBuffer newBuffer = getBuffer(true /* input */, index);
   3055         synchronized(mBufferLock) {
   3056             invalidateByteBuffer(mCachedInputBuffers, index);
   3057             mDequeuedInputBuffers.put(index, newBuffer);
   3058         }
   3059         return newBuffer;
   3060     }
   3061 
   3062     /**
   3063      * Returns a writable Image object for a dequeued input buffer
   3064      * index to contain the raw input video frame.
   3065      *
   3066      * After calling this method any ByteBuffer or Image object
   3067      * previously returned for the same input index MUST no longer
   3068      * be used.
   3069      *
   3070      * @param index The index of a client-owned input buffer previously
   3071      *              returned from a call to {@link #dequeueInputBuffer},
   3072      *              or received via an onInputBufferAvailable callback.
   3073      *
   3074      * @return the input image, or null if the index is not a
   3075      * dequeued input buffer, or not a ByteBuffer that contains a
   3076      * raw image.
   3077      *
   3078      * @throws IllegalStateException if not in the Executing state.
   3079      * @throws MediaCodec.CodecException upon codec error.
   3080      */
   3081     @Nullable
   3082     public Image getInputImage(int index) {
   3083         Image newImage = getImage(true /* input */, index);
   3084         synchronized(mBufferLock) {
   3085             invalidateByteBuffer(mCachedInputBuffers, index);
   3086             mDequeuedInputBuffers.put(index, newImage);
   3087         }
   3088         return newImage;
   3089     }
   3090 
   3091     /**
   3092      * Returns a read-only ByteBuffer for a dequeued output buffer
   3093      * index. The position and limit of the returned buffer are set
   3094      * to the valid output data.
   3095      *
   3096      * After calling this method, any ByteBuffer or Image object
   3097      * previously returned for the same output index MUST no longer
   3098      * be used.
   3099      *
   3100      * @param index The index of a client-owned output buffer previously
   3101      *              returned from a call to {@link #dequeueOutputBuffer},
   3102      *              or received via an onOutputBufferAvailable callback.
   3103      *
   3104      * @return the output buffer, or null if the index is not a dequeued
   3105      * output buffer, or the codec is configured with an output surface.
   3106      *
   3107      * @throws IllegalStateException if not in the Executing state.
   3108      * @throws MediaCodec.CodecException upon codec error.
   3109      */
   3110     @Nullable
   3111     public ByteBuffer getOutputBuffer(int index) {
   3112         ByteBuffer newBuffer = getBuffer(false /* input */, index);
   3113         synchronized(mBufferLock) {
   3114             invalidateByteBuffer(mCachedOutputBuffers, index);
   3115             mDequeuedOutputBuffers.put(index, newBuffer);
   3116         }
   3117         return newBuffer;
   3118     }
   3119 
   3120     /**
   3121      * Returns a read-only Image object for a dequeued output buffer
   3122      * index that contains the raw video frame.
   3123      *
   3124      * After calling this method, any ByteBuffer or Image object previously
   3125      * returned for the same output index MUST no longer be used.
   3126      *
   3127      * @param index The index of a client-owned output buffer previously
   3128      *              returned from a call to {@link #dequeueOutputBuffer},
   3129      *              or received via an onOutputBufferAvailable callback.
   3130      *
   3131      * @return the output image, or null if the index is not a
   3132      * dequeued output buffer, not a raw video frame, or if the codec
   3133      * was configured with an output surface.
   3134      *
   3135      * @throws IllegalStateException if not in the Executing state.
   3136      * @throws MediaCodec.CodecException upon codec error.
   3137      */
   3138     @Nullable
   3139     public Image getOutputImage(int index) {
   3140         Image newImage = getImage(false /* input */, index);
   3141         synchronized(mBufferLock) {
   3142             invalidateByteBuffer(mCachedOutputBuffers, index);
   3143             mDequeuedOutputBuffers.put(index, newImage);
   3144         }
   3145         return newImage;
   3146     }
   3147 
   3148     /**
   3149      * The content is scaled to the surface dimensions
   3150      */
   3151     public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT               = 1;
   3152 
   3153     /**
   3154      * The content is scaled, maintaining its aspect ratio, the whole
   3155      * surface area is used, content may be cropped.
   3156      * <p class=note>
   3157      * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot
   3158      * configure the pixel aspect ratio for a {@link Surface}.
   3159      * <p class=note>
   3160      * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if
   3161      * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees.
   3162      */
   3163     public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2;
   3164 
   3165     /** @hide */
   3166     @IntDef({
   3167         VIDEO_SCALING_MODE_SCALE_TO_FIT,
   3168         VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING,
   3169     })
   3170     @Retention(RetentionPolicy.SOURCE)
   3171     public @interface VideoScalingMode {}
   3172 
   3173     /**
   3174      * If a surface has been specified in a previous call to {@link #configure}
   3175      * specifies the scaling mode to use. The default is "scale to fit".
   3176      * <p class=note>
   3177      * The scaling mode may be reset to the <strong>default</strong> each time an
   3178      * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client
   3179      * must call this method after every buffer change event (and before the first output buffer is
   3180      * released for rendering) to ensure consistent scaling mode.
   3181      * <p class=note>
   3182      * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done
   3183      * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event.
   3184      *
   3185      * @throws IllegalArgumentException if mode is not recognized.
   3186      * @throws IllegalStateException if in the Released state.
   3187      */
   3188     public native final void setVideoScalingMode(@VideoScalingMode int mode);
   3189 
   3190     /**
   3191      * Get the component name. If the codec was created by createDecoderByType
   3192      * or createEncoderByType, what component is chosen is not known beforehand.
   3193      * @throws IllegalStateException if in the Released state.
   3194      */
   3195     @NonNull
   3196     public native final String getName();
   3197 
   3198     /**
   3199      *  Return Metrics data about the current codec instance.
   3200      *
   3201      * @return a {@link PersistableBundle} containing the set of attributes and values
   3202      * available for the media being handled by this instance of MediaCodec
   3203      * The attributes are descibed in {@link MetricsConstants}.
   3204      *
   3205      * Additional vendor-specific fields may also be present in
   3206      * the return value.
   3207      */
   3208     public PersistableBundle getMetrics() {
   3209         PersistableBundle bundle = native_getMetrics();
   3210         return bundle;
   3211     }
   3212 
   3213     private native PersistableBundle native_getMetrics();
   3214 
   3215     /**
   3216      * Change a video encoder's target bitrate on the fly. The value is an
   3217      * Integer object containing the new bitrate in bps.
   3218      */
   3219     public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate";
   3220 
   3221     /**
   3222      * Temporarily suspend/resume encoding of input data. While suspended
   3223      * input data is effectively discarded instead of being fed into the
   3224      * encoder. This parameter really only makes sense to use with an encoder
   3225      * in "surface-input" mode, as the client code has no control over the
   3226      * input-side of the encoder in that case.
   3227      * The value is an Integer object containing the value 1 to suspend
   3228      * or the value 0 to resume.
   3229      */
   3230     public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames";
   3231 
   3232     /**
   3233      * Request that the encoder produce a sync frame "soon".
   3234      * Provide an Integer with the value 0.
   3235      */
   3236     public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync";
   3237 
   3238     /**
   3239      * Communicate additional parameter changes to the component instance.
   3240      * <b>Note:</b> Some of these parameter changes may silently fail to apply.
   3241      *
   3242      * @param params The bundle of parameters to set.
   3243      * @throws IllegalStateException if in the Released state.
   3244      */
   3245     public final void setParameters(@Nullable Bundle params) {
   3246         if (params == null) {
   3247             return;
   3248         }
   3249 
   3250         String[] keys = new String[params.size()];
   3251         Object[] values = new Object[params.size()];
   3252 
   3253         int i = 0;
   3254         for (final String key: params.keySet()) {
   3255             keys[i] = key;
   3256             values[i] = params.get(key);
   3257             ++i;
   3258         }
   3259 
   3260         setParameters(keys, values);
   3261     }
   3262 
   3263     /**
   3264      * Sets an asynchronous callback for actionable MediaCodec events.
   3265      *
   3266      * If the client intends to use the component in asynchronous mode,
   3267      * a valid callback should be provided before {@link #configure} is called.
   3268      *
   3269      * When asynchronous callback is enabled, the client should not call
   3270      * {@link #getInputBuffers}, {@link #getOutputBuffers},
   3271      * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}.
   3272      * <p>
   3273      * Also, {@link #flush} behaves differently in asynchronous mode.  After calling
   3274      * {@code flush}, you must call {@link #start} to "resume" receiving input buffers,
   3275      * even if an input surface was created.
   3276      *
   3277      * @param cb The callback that will run.  Use {@code null} to clear a previously
   3278      *           set callback (before {@link #configure configure} is called and run
   3279      *           in synchronous mode).
   3280      * @param handler Callbacks will happen on the handler's thread. If {@code null},
   3281      *           callbacks are done on the default thread (the caller's thread or the
   3282      *           main thread.)
   3283      */
   3284     public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) {
   3285         if (cb != null) {
   3286             synchronized (mListenerLock) {
   3287                 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler);
   3288                 // NOTE: there are no callbacks on the handler at this time, but check anyways
   3289                 // even if we were to extend this to be callable dynamically, it must
   3290                 // be called when codec is flushed, so no messages are pending.
   3291                 if (newHandler != mCallbackHandler) {
   3292                     mCallbackHandler.removeMessages(EVENT_SET_CALLBACK);
   3293                     mCallbackHandler.removeMessages(EVENT_CALLBACK);
   3294                     mCallbackHandler = newHandler;
   3295                 }
   3296             }
   3297         } else if (mCallbackHandler != null) {
   3298             mCallbackHandler.removeMessages(EVENT_SET_CALLBACK);
   3299             mCallbackHandler.removeMessages(EVENT_CALLBACK);
   3300         }
   3301 
   3302         if (mCallbackHandler != null) {
   3303             // set java callback on main handler
   3304             Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb);
   3305             mCallbackHandler.sendMessage(msg);
   3306 
   3307             // set native handler here, don't post to handler because
   3308             // it may cause the callback to be delayed and set in a wrong state.
   3309             // Note that native codec may start sending events to the callback
   3310             // handler after this returns.
   3311             native_setCallback(cb);
   3312         }
   3313     }
   3314 
   3315     /**
   3316      * Sets an asynchronous callback for actionable MediaCodec events on the default
   3317      * looper.
   3318      * <p>
   3319      * Same as {@link #setCallback(Callback, Handler)} with handler set to null.
   3320      * @param cb The callback that will run.  Use {@code null} to clear a previously
   3321      *           set callback (before {@link #configure configure} is called and run
   3322      *           in synchronous mode).
   3323      * @see #setCallback(Callback, Handler)
   3324      */
   3325     public void setCallback(@Nullable /* MediaCodec. */ Callback cb) {
   3326         setCallback(cb, null /* handler */);
   3327     }
   3328 
   3329     /**
   3330      * Listener to be called when an output frame has rendered on the output surface
   3331      *
   3332      * @see MediaCodec#setOnFrameRenderedListener
   3333      */
   3334     public interface OnFrameRenderedListener {
   3335 
   3336         /**
   3337          * Called when an output frame has rendered on the output surface.
   3338          * <p>
   3339          * <strong>Note:</strong> This callback is for informational purposes only: to get precise
   3340          * render timing samples, and can be significantly delayed and batched. Some frames may have
   3341          * been rendered even if there was no callback generated.
   3342          *
   3343          * @param codec the MediaCodec instance
   3344          * @param presentationTimeUs the presentation time (media time) of the frame rendered.
   3345          *          This is usually the same as specified in {@link #queueInputBuffer}; however,
   3346          *          some codecs may alter the media time by applying some time-based transformation,
   3347          *          such as frame rate conversion. In that case, presentation time corresponds
   3348          *          to the actual output frame rendered.
   3349          * @param nanoTime The system time when the frame was rendered.
   3350          *
   3351          * @see System#nanoTime
   3352          */
   3353         public void onFrameRendered(
   3354                 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime);
   3355     }
   3356 
   3357     /**
   3358      * Registers a callback to be invoked when an output frame is rendered on the output surface.
   3359      * <p>
   3360      * This method can be called in any codec state, but will only have an effect in the
   3361      * Executing state for codecs that render buffers to the output surface.
   3362      * <p>
   3363      * <strong>Note:</strong> This callback is for informational purposes only: to get precise
   3364      * render timing samples, and can be significantly delayed and batched. Some frames may have
   3365      * been rendered even if there was no callback generated.
   3366      *
   3367      * @param listener the callback that will be run
   3368      * @param handler the callback will be run on the handler's thread. If {@code null},
   3369      *           the callback will be run on the default thread, which is the looper
   3370      *           from which the codec was created, or a new thread if there was none.
   3371      */
   3372     public void setOnFrameRenderedListener(
   3373             @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) {
   3374         synchronized (mListenerLock) {
   3375             mOnFrameRenderedListener = listener;
   3376             if (listener != null) {
   3377                 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler);
   3378                 if (newHandler != mOnFrameRenderedHandler) {
   3379                     mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED);
   3380                 }
   3381                 mOnFrameRenderedHandler = newHandler;
   3382             } else if (mOnFrameRenderedHandler != null) {
   3383                 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED);
   3384             }
   3385             native_enableOnFrameRenderedListener(listener != null);
   3386         }
   3387     }
   3388 
   3389     private native void native_enableOnFrameRenderedListener(boolean enable);
   3390 
   3391     private EventHandler getEventHandlerOn(
   3392             @Nullable Handler handler, @NonNull EventHandler lastHandler) {
   3393         if (handler == null) {
   3394             return mEventHandler;
   3395         } else {
   3396             Looper looper = handler.getLooper();
   3397             if (lastHandler.getLooper() == looper) {
   3398                 return lastHandler;
   3399             } else {
   3400                 return new EventHandler(this, looper);
   3401             }
   3402         }
   3403     }
   3404 
   3405     /**
   3406      * MediaCodec callback interface. Used to notify the user asynchronously
   3407      * of various MediaCodec events.
   3408      */
   3409     public static abstract class Callback {
   3410         /**
   3411          * Called when an input buffer becomes available.
   3412          *
   3413          * @param codec The MediaCodec object.
   3414          * @param index The index of the available input buffer.
   3415          */
   3416         public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index);
   3417 
   3418         /**
   3419          * Called when an output buffer becomes available.
   3420          *
   3421          * @param codec The MediaCodec object.
   3422          * @param index The index of the available output buffer.
   3423          * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}.
   3424          */
   3425         public abstract void onOutputBufferAvailable(
   3426                 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info);
   3427 
   3428         /**
   3429          * Called when the MediaCodec encountered an error
   3430          *
   3431          * @param codec The MediaCodec object.
   3432          * @param e The {@link MediaCodec.CodecException} object describing the error.
   3433          */
   3434         public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e);
   3435 
   3436         /**
   3437          * Called when the output format has changed
   3438          *
   3439          * @param codec The MediaCodec object.
   3440          * @param format The new output format.
   3441          */
   3442         public abstract void onOutputFormatChanged(
   3443                 @NonNull MediaCodec codec, @NonNull MediaFormat format);
   3444     }
   3445 
   3446     private void postEventFromNative(
   3447             int what, int arg1, int arg2, @Nullable Object obj) {
   3448         synchronized (mListenerLock) {
   3449             EventHandler handler = mEventHandler;
   3450             if (what == EVENT_CALLBACK) {
   3451                 handler = mCallbackHandler;
   3452             } else if (what == EVENT_FRAME_RENDERED) {
   3453                 handler = mOnFrameRenderedHandler;
   3454             }
   3455             if (handler != null) {
   3456                 Message msg = handler.obtainMessage(what, arg1, arg2, obj);
   3457                 handler.sendMessage(msg);
   3458             }
   3459         }
   3460     }
   3461 
   3462     private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values);
   3463 
   3464     /**
   3465      * Get the codec info. If the codec was created by createDecoderByType
   3466      * or createEncoderByType, what component is chosen is not known beforehand,
   3467      * and thus the caller does not have the MediaCodecInfo.
   3468      * @throws IllegalStateException if in the Released state.
   3469      */
   3470     @NonNull
   3471     public MediaCodecInfo getCodecInfo() {
   3472         return MediaCodecList.getInfoFor(getName());
   3473     }
   3474 
   3475     @NonNull
   3476     private native final ByteBuffer[] getBuffers(boolean input);
   3477 
   3478     @Nullable
   3479     private native final ByteBuffer getBuffer(boolean input, int index);
   3480 
   3481     @Nullable
   3482     private native final Image getImage(boolean input, int index);
   3483 
   3484     private static native final void native_init();
   3485 
   3486     private native final void native_setup(
   3487             @NonNull String name, boolean nameIsType, boolean encoder);
   3488 
   3489     private native final void native_finalize();
   3490 
   3491     static {
   3492         System.loadLibrary("media_jni");
   3493         native_init();
   3494     }
   3495 
   3496     private long mNativeContext;
   3497 
   3498     /** @hide */
   3499     public static class MediaImage extends Image {
   3500         private final boolean mIsReadOnly;
   3501         private final int mWidth;
   3502         private final int mHeight;
   3503         private final int mFormat;
   3504         private long mTimestamp;
   3505         private final Plane[] mPlanes;
   3506         private final ByteBuffer mBuffer;
   3507         private final ByteBuffer mInfo;
   3508         private final int mXOffset;
   3509         private final int mYOffset;
   3510 
   3511         private final static int TYPE_YUV = 1;
   3512 
   3513         @Override
   3514         public int getFormat() {
   3515             throwISEIfImageIsInvalid();
   3516             return mFormat;
   3517         }
   3518 
   3519         @Override
   3520         public int getHeight() {
   3521             throwISEIfImageIsInvalid();
   3522             return mHeight;
   3523         }
   3524 
   3525         @Override
   3526         public int getWidth() {
   3527             throwISEIfImageIsInvalid();
   3528             return mWidth;
   3529         }
   3530 
   3531         @Override
   3532         public long getTimestamp() {
   3533             throwISEIfImageIsInvalid();
   3534             return mTimestamp;
   3535         }
   3536 
   3537         @Override
   3538         @NonNull
   3539         public Plane[] getPlanes() {
   3540             throwISEIfImageIsInvalid();
   3541             return Arrays.copyOf(mPlanes, mPlanes.length);
   3542         }
   3543 
   3544         @Override
   3545         public void close() {
   3546             if (mIsImageValid) {
   3547                 java.nio.NioUtils.freeDirectBuffer(mBuffer);
   3548                 mIsImageValid = false;
   3549             }
   3550         }
   3551 
   3552         /**
   3553          * Set the crop rectangle associated with this frame.
   3554          * <p>
   3555          * The crop rectangle specifies the region of valid pixels in the image,
   3556          * using coordinates in the largest-resolution plane.
   3557          */
   3558         @Override
   3559         public void setCropRect(@Nullable Rect cropRect) {
   3560             if (mIsReadOnly) {
   3561                 throw new ReadOnlyBufferException();
   3562             }
   3563             super.setCropRect(cropRect);
   3564         }
   3565 
   3566 
   3567         public MediaImage(
   3568                 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly,
   3569                 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) {
   3570             mFormat = ImageFormat.YUV_420_888;
   3571             mTimestamp = timestamp;
   3572             mIsImageValid = true;
   3573             mIsReadOnly = buffer.isReadOnly();
   3574             mBuffer = buffer.duplicate();
   3575 
   3576             // save offsets and info
   3577             mXOffset = xOffset;
   3578             mYOffset = yOffset;
   3579             mInfo = info;
   3580 
   3581             // read media-info.  See MediaImage2
   3582             if (info.remaining() == 104) {
   3583                 int type = info.getInt();
   3584                 if (type != TYPE_YUV) {
   3585                     throw new UnsupportedOperationException("unsupported type: " + type);
   3586                 }
   3587                 int numPlanes = info.getInt();
   3588                 if (numPlanes != 3) {
   3589                     throw new RuntimeException("unexpected number of planes: " + numPlanes);
   3590                 }
   3591                 mWidth = info.getInt();
   3592                 mHeight = info.getInt();
   3593                 if (mWidth < 1 || mHeight < 1) {
   3594                     throw new UnsupportedOperationException(
   3595                             "unsupported size: " + mWidth + "x" + mHeight);
   3596                 }
   3597                 int bitDepth = info.getInt();
   3598                 if (bitDepth != 8) {
   3599                     throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth);
   3600                 }
   3601                 int bitDepthAllocated = info.getInt();
   3602                 if (bitDepthAllocated != 8) {
   3603                     throw new UnsupportedOperationException(
   3604                             "unsupported allocated bit depth: " + bitDepthAllocated);
   3605                 }
   3606                 mPlanes = new MediaPlane[numPlanes];
   3607                 for (int ix = 0; ix < numPlanes; ix++) {
   3608                     int planeOffset = info.getInt();
   3609                     int colInc = info.getInt();
   3610                     int rowInc = info.getInt();
   3611                     int horiz = info.getInt();
   3612                     int vert = info.getInt();
   3613                     if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) {
   3614                         throw new UnsupportedOperationException("unexpected subsampling: "
   3615                                 + horiz + "x" + vert + " on plane " + ix);
   3616                     }
   3617                     if (colInc < 1 || rowInc < 1) {
   3618                         throw new UnsupportedOperationException("unexpected strides: "
   3619                                 + colInc + " pixel, " + rowInc + " row on plane " + ix);
   3620                     }
   3621 
   3622                     buffer.clear();
   3623                     buffer.position(mBuffer.position() + planeOffset
   3624                             + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc);
   3625                     buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8)
   3626                             + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc);
   3627                     mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc);
   3628                 }
   3629             } else {
   3630                 throw new UnsupportedOperationException(
   3631                         "unsupported info length: " + info.remaining());
   3632             }
   3633 
   3634             if (cropRect == null) {
   3635                 cropRect = new Rect(0, 0, mWidth, mHeight);
   3636             }
   3637             cropRect.offset(-xOffset, -yOffset);
   3638             super.setCropRect(cropRect);
   3639         }
   3640 
   3641         private class MediaPlane extends Plane {
   3642             public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) {
   3643                 mData = buffer;
   3644                 mRowInc = rowInc;
   3645                 mColInc = colInc;
   3646             }
   3647 
   3648             @Override
   3649             public int getRowStride() {
   3650                 throwISEIfImageIsInvalid();
   3651                 return mRowInc;
   3652             }
   3653 
   3654             @Override
   3655             public int getPixelStride() {
   3656                 throwISEIfImageIsInvalid();
   3657                 return mColInc;
   3658             }
   3659 
   3660             @Override
   3661             @NonNull
   3662             public ByteBuffer getBuffer() {
   3663                 throwISEIfImageIsInvalid();
   3664                 return mData;
   3665             }
   3666 
   3667             private final int mRowInc;
   3668             private final int mColInc;
   3669             private final ByteBuffer mData;
   3670         }
   3671     }
   3672 
   3673     public final static class MetricsConstants
   3674     {
   3675         private MetricsConstants() {}
   3676 
   3677         /**
   3678          * Key to extract the codec being used
   3679          * from the {@link MediaCodec#getMetrics} return value.
   3680          * The value is a String.
   3681          */
   3682         public static final String CODEC = "android.media.mediacodec.codec";
   3683 
   3684         /**
   3685          * Key to extract the MIME type
   3686          * from the {@link MediaCodec#getMetrics} return value.
   3687          * The value is a String.
   3688          */
   3689         public static final String MIME_TYPE = "android.media.mediacodec.mime";
   3690 
   3691         /**
   3692          * Key to extract what the codec mode
   3693          * from the {@link MediaCodec#getMetrics} return value.
   3694          * The value is a String. Values will be one of the constants
   3695          * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}.
   3696          */
   3697         public static final String MODE = "android.media.mediacodec.mode";
   3698 
   3699         /**
   3700          * The value returned for the key {@link #MODE} when the
   3701          * codec is a audio codec.
   3702          */
   3703         public static final String MODE_AUDIO = "audio";
   3704 
   3705         /**
   3706          * The value returned for the key {@link #MODE} when the
   3707          * codec is a video codec.
   3708          */
   3709         public static final String MODE_VIDEO = "video";
   3710 
   3711         /**
   3712          * Key to extract the flag indicating whether the codec is running
   3713          * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value.
   3714          * The value is an integer.
   3715          * A 0 indicates decoder; 1 indicates encoder.
   3716          */
   3717         public static final String ENCODER = "android.media.mediacodec.encoder";
   3718 
   3719         /**
   3720          * Key to extract the flag indicating whether the codec is running
   3721          * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value.
   3722          * The value is an integer.
   3723          */
   3724         public static final String SECURE = "android.media.mediacodec.secure";
   3725 
   3726         /**
   3727          * Key to extract the width (in pixels) of the video track
   3728          * from the {@link MediaCodec#getMetrics} return value.
   3729          * The value is an integer.
   3730          */
   3731         public static final String WIDTH = "android.media.mediacodec.width";
   3732 
   3733         /**
   3734          * Key to extract the height (in pixels) of the video track
   3735          * from the {@link MediaCodec#getMetrics} return value.
   3736          * The value is an integer.
   3737          */
   3738         public static final String HEIGHT = "android.media.mediacodec.height";
   3739 
   3740         /**
   3741          * Key to extract the rotation (in degrees) to properly orient the video
   3742          * from the {@link MediaCodec#getMetrics} return.
   3743          * The value is a integer.
   3744          */
   3745         public static final String ROTATION = "android.media.mediacodec.rotation";
   3746 
   3747     }
   3748 }
   3749