1 /* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package androidx.media.filterfw.decoder; 17 18 import android.annotation.TargetApi; 19 import android.graphics.SurfaceTexture; 20 import android.media.MediaCodec; 21 import android.media.MediaCodec.BufferInfo; 22 import android.media.MediaCodecInfo; 23 import android.media.MediaCodecInfo.CodecCapabilities; 24 import android.media.MediaCodecList; 25 import android.media.MediaFormat; 26 import android.util.SparseIntArray; 27 import androidx.media.filterfw.ColorSpace; 28 import androidx.media.filterfw.Frame; 29 import androidx.media.filterfw.FrameImage2D; 30 import androidx.media.filterfw.PixelUtils; 31 32 import java.io.IOException; 33 import java.nio.ByteBuffer; 34 import java.util.Arrays; 35 import java.util.HashSet; 36 import java.util.Set; 37 import java.util.TreeMap; 38 39 /** 40 * {@link TrackDecoder} that decodes a video track and renders the frames onto a 41 * {@link SurfaceTexture}. 42 * 43 * This implementation purely uses CPU based methods to decode and color-convert the frames. 44 */ 45 @TargetApi(16) 46 public class CpuVideoTrackDecoder extends VideoTrackDecoder { 47 48 private static final int COLOR_FORMAT_UNSET = -1; 49 50 private final int mWidth; 51 private final int mHeight; 52 53 private int mColorFormat = COLOR_FORMAT_UNSET; 54 private long mCurrentPresentationTimeUs; 55 private ByteBuffer mDecodedBuffer; 56 private ByteBuffer mUnrotatedBytes; 57 58 protected CpuVideoTrackDecoder(int trackIndex, MediaFormat format, Listener listener) { 59 super(trackIndex, format, listener); 60 61 mWidth = format.getInteger(MediaFormat.KEY_WIDTH); 62 mHeight = format.getInteger(MediaFormat.KEY_HEIGHT); 63 } 64 65 @Override 66 protected MediaCodec initMediaCodec(MediaFormat format) { 67 // Find a codec for our video that can output to one of our supported color-spaces 68 MediaCodec mediaCodec = findDecoderCodec(format, new int[] { 69 CodecCapabilities.COLOR_Format32bitARGB8888, 70 CodecCapabilities.COLOR_FormatYUV420Planar}); 71 if (mediaCodec == null) { 72 throw new RuntimeException( 73 "Could not find a suitable decoder for format: " + format + "!"); 74 } 75 mediaCodec.configure(format, null, null, 0); 76 return mediaCodec; 77 } 78 79 @Override 80 protected boolean onDataAvailable( 81 MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) { 82 83 mCurrentPresentationTimeUs = info.presentationTimeUs; 84 mDecodedBuffer = buffers[bufferIndex]; 85 86 if (mColorFormat == -1) { 87 mColorFormat = codec.getOutputFormat().getInteger(MediaFormat.KEY_COLOR_FORMAT); 88 } 89 90 markFrameAvailable(); 91 notifyListener(); 92 93 // Wait for the grab before we release this buffer. 94 waitForFrameGrab(); 95 96 codec.releaseOutputBuffer(bufferIndex, false); 97 98 return false; 99 } 100 101 @Override 102 protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) { 103 // Calculate output dimensions 104 int outputWidth = mWidth; 105 int outputHeight = mHeight; 106 if (needSwapDimension(rotation)) { 107 outputWidth = mHeight; 108 outputHeight = mWidth; 109 } 110 111 // Create output frame 112 outputVideoFrame.resize(new int[] {outputWidth, outputHeight}); 113 outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000); 114 ByteBuffer outBytes = outputVideoFrame.lockBytes(Frame.MODE_WRITE); 115 116 // Set data 117 if (rotation == MediaDecoder.ROTATE_NONE) { 118 convertImage(mDecodedBuffer, outBytes, mColorFormat, mWidth, mHeight); 119 } else { 120 if (mUnrotatedBytes == null) { 121 mUnrotatedBytes = ByteBuffer.allocateDirect(mWidth * mHeight * 4); 122 } 123 // TODO: This could be optimized by including the rotation in the color conversion. 124 convertImage(mDecodedBuffer, mUnrotatedBytes, mColorFormat, mWidth, mHeight); 125 copyRotate(mUnrotatedBytes, outBytes, rotation); 126 } 127 outputVideoFrame.unlock(); 128 } 129 130 /** 131 * Copy the input data to the output data applying the specified rotation. 132 * 133 * @param input The input image data 134 * @param output Buffer for the output image data 135 * @param rotation The rotation to apply 136 */ 137 private void copyRotate(ByteBuffer input, ByteBuffer output, int rotation) { 138 int offset; 139 int pixStride; 140 int rowStride; 141 switch (rotation) { 142 case MediaDecoder.ROTATE_NONE: 143 offset = 0; 144 pixStride = 1; 145 rowStride = mWidth; 146 break; 147 case MediaDecoder.ROTATE_90_LEFT: 148 offset = (mWidth - 1) * mHeight; 149 pixStride = -mHeight; 150 rowStride = 1; 151 break; 152 case MediaDecoder.ROTATE_90_RIGHT: 153 offset = mHeight - 1; 154 pixStride = mHeight; 155 rowStride = -1; 156 break; 157 case MediaDecoder.ROTATE_180: 158 offset = mWidth * mHeight - 1; 159 pixStride = -1; 160 rowStride = -mWidth; 161 break; 162 default: 163 throw new IllegalArgumentException("Unsupported rotation " + rotation + "!"); 164 } 165 PixelUtils.copyPixels(input, output, mWidth, mHeight, offset, pixStride, rowStride); 166 } 167 168 /** 169 * Looks for a codec with the specified requirements. 170 * 171 * The set of codecs will be filtered down to those that meet the following requirements: 172 * <ol> 173 * <li>The codec is a decoder.</li> 174 * <li>The codec can decode a video of the specified format.</li> 175 * <li>The codec can decode to one of the specified color formats.</li> 176 * </ol> 177 * If multiple codecs are found, the one with the preferred color-format is taken. Color format 178 * preference is determined by the order of their appearance in the color format array. 179 * 180 * @param format The format the codec must decode. 181 * @param requiredColorFormats Array of target color spaces ordered by preference. 182 * @return A codec that meets the requirements, or null if no such codec was found. 183 */ 184 private static MediaCodec findDecoderCodec(MediaFormat format, int[] requiredColorFormats) { 185 TreeMap<Integer, String> candidateCodecs = new TreeMap<Integer, String>(); 186 SparseIntArray colorPriorities = intArrayToPriorityMap(requiredColorFormats); 187 for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { 188 // Get next codec 189 MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i); 190 191 // Check that this is a decoder 192 if (info.isEncoder()) { 193 continue; 194 } 195 196 // Check if this codec can decode the video in question 197 String requiredType = format.getString(MediaFormat.KEY_MIME); 198 String[] supportedTypes = info.getSupportedTypes(); 199 Set<String> typeSet = new HashSet<String>(Arrays.asList(supportedTypes)); 200 201 // Check if it can decode to one of the required color formats 202 if (typeSet.contains(requiredType)) { 203 CodecCapabilities capabilities = info.getCapabilitiesForType(requiredType); 204 for (int supportedColorFormat : capabilities.colorFormats) { 205 if (colorPriorities.indexOfKey(supportedColorFormat) >= 0) { 206 int priority = colorPriorities.get(supportedColorFormat); 207 candidateCodecs.put(priority, info.getName()); 208 } 209 } 210 } 211 } 212 213 // Pick the best codec (with the highest color priority) 214 if (candidateCodecs.isEmpty()) { 215 return null; 216 } else { 217 String bestCodec = candidateCodecs.firstEntry().getValue(); 218 try { 219 return MediaCodec.createByCodecName(bestCodec); 220 } catch (IOException e) { 221 throw new RuntimeException( 222 "failed to create codec for " 223 + bestCodec, e); 224 } 225 } 226 } 227 228 private static SparseIntArray intArrayToPriorityMap(int[] values) { 229 SparseIntArray result = new SparseIntArray(); 230 for (int priority = 0; priority < values.length; ++priority) { 231 result.append(values[priority], priority); 232 } 233 return result; 234 } 235 236 private static void convertImage( 237 ByteBuffer input, ByteBuffer output, int colorFormat, int width, int height) { 238 switch (colorFormat) { 239 case CodecCapabilities.COLOR_Format32bitARGB8888: 240 ColorSpace.convertArgb8888ToRgba8888(input, output, width, height); 241 break; 242 case CodecCapabilities.COLOR_FormatYUV420Planar: 243 ColorSpace.convertYuv420pToRgba8888(input, output, width, height); 244 break; 245 default: 246 throw new RuntimeException("Unsupported color format: " + colorFormat + "!"); 247 } 248 } 249 250 } 251
