1 /* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package androidx.media.filterfw.decoder; 17 18 import android.annotation.TargetApi; 19 import android.graphics.SurfaceTexture; 20 import android.media.MediaCodec; 21 import android.media.MediaCodec.BufferInfo; 22 import android.media.MediaCodecInfo; 23 import android.media.MediaCodecInfo.CodecCapabilities; 24 import android.media.MediaCodecList; 25 import android.media.MediaFormat; 26 import android.util.SparseIntArray; 27 import androidx.media.filterfw.ColorSpace; 28 import androidx.media.filterfw.Frame; 29 import androidx.media.filterfw.FrameImage2D; 30 import androidx.media.filterfw.PixelUtils; 31 32 import java.nio.ByteBuffer; 33 import java.util.Arrays; 34 import java.util.HashSet; 35 import java.util.Set; 36 import java.util.TreeMap; 37 38 /** 39 * {@link TrackDecoder} that decodes a video track and renders the frames onto a 40 * {@link SurfaceTexture}. 41 * 42 * This implementation purely uses CPU based methods to decode and color-convert the frames. 43 */ 44 @TargetApi(16) 45 public class CpuVideoTrackDecoder extends VideoTrackDecoder { 46 47 private static final int COLOR_FORMAT_UNSET = -1; 48 49 private final int mWidth; 50 private final int mHeight; 51 52 private int mColorFormat = COLOR_FORMAT_UNSET; 53 private long mCurrentPresentationTimeUs; 54 private ByteBuffer mDecodedBuffer; 55 private ByteBuffer mUnrotatedBytes; 56 57 protected CpuVideoTrackDecoder(int trackIndex, MediaFormat format, Listener listener) { 58 super(trackIndex, format, listener); 59 60 mWidth = format.getInteger(MediaFormat.KEY_WIDTH); 61 mHeight = format.getInteger(MediaFormat.KEY_HEIGHT); 62 } 63 64 @Override 65 protected MediaCodec initMediaCodec(MediaFormat format) { 66 // Find a codec for our video that can output to one of our supported color-spaces 67 MediaCodec mediaCodec = findDecoderCodec(format, new int[] { 68 CodecCapabilities.COLOR_Format32bitARGB8888, 69 CodecCapabilities.COLOR_FormatYUV420Planar}); 70 if (mediaCodec == null) { 71 throw new RuntimeException( 72 "Could not find a suitable decoder for format: " + format + "!"); 73 } 74 mediaCodec.configure(format, null, null, 0); 75 return mediaCodec; 76 } 77 78 @Override 79 protected boolean onDataAvailable( 80 MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) { 81 82 mCurrentPresentationTimeUs = info.presentationTimeUs; 83 mDecodedBuffer = buffers[bufferIndex]; 84 85 if (mColorFormat == -1) { 86 mColorFormat = codec.getOutputFormat().getInteger(MediaFormat.KEY_COLOR_FORMAT); 87 } 88 89 markFrameAvailable(); 90 notifyListener(); 91 92 // Wait for the grab before we release this buffer. 93 waitForFrameGrab(); 94 95 codec.releaseOutputBuffer(bufferIndex, false); 96 97 return false; 98 } 99 100 @Override 101 protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) { 102 // Calculate output dimensions 103 int outputWidth = mWidth; 104 int outputHeight = mHeight; 105 if (needSwapDimension(rotation)) { 106 outputWidth = mHeight; 107 outputHeight = mWidth; 108 } 109 110 // Create output frame 111 outputVideoFrame.resize(new int[] {outputWidth, outputHeight}); 112 outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000); 113 ByteBuffer outBytes = outputVideoFrame.lockBytes(Frame.MODE_WRITE); 114 115 // Set data 116 if (rotation == MediaDecoder.ROTATE_NONE) { 117 convertImage(mDecodedBuffer, outBytes, mColorFormat, mWidth, mHeight); 118 } else { 119 if (mUnrotatedBytes == null) { 120 mUnrotatedBytes = ByteBuffer.allocateDirect(mWidth * mHeight * 4); 121 } 122 // TODO: This could be optimized by including the rotation in the color conversion. 123 convertImage(mDecodedBuffer, mUnrotatedBytes, mColorFormat, mWidth, mHeight); 124 copyRotate(mUnrotatedBytes, outBytes, rotation); 125 } 126 outputVideoFrame.unlock(); 127 } 128 129 /** 130 * Copy the input data to the output data applying the specified rotation. 131 * 132 * @param input The input image data 133 * @param output Buffer for the output image data 134 * @param rotation The rotation to apply 135 */ 136 private void copyRotate(ByteBuffer input, ByteBuffer output, int rotation) { 137 int offset; 138 int pixStride; 139 int rowStride; 140 switch (rotation) { 141 case MediaDecoder.ROTATE_NONE: 142 offset = 0; 143 pixStride = 1; 144 rowStride = mWidth; 145 break; 146 case MediaDecoder.ROTATE_90_LEFT: 147 offset = (mWidth - 1) * mHeight; 148 pixStride = -mHeight; 149 rowStride = 1; 150 break; 151 case MediaDecoder.ROTATE_90_RIGHT: 152 offset = mHeight - 1; 153 pixStride = mHeight; 154 rowStride = -1; 155 break; 156 case MediaDecoder.ROTATE_180: 157 offset = mWidth * mHeight - 1; 158 pixStride = -1; 159 rowStride = -mWidth; 160 break; 161 default: 162 throw new IllegalArgumentException("Unsupported rotation " + rotation + "!"); 163 } 164 PixelUtils.copyPixels(input, output, mWidth, mHeight, offset, pixStride, rowStride); 165 } 166 167 /** 168 * Looks for a codec with the specified requirements. 169 * 170 * The set of codecs will be filtered down to those that meet the following requirements: 171 * <ol> 172 * <li>The codec is a decoder.</li> 173 * <li>The codec can decode a video of the specified format.</li> 174 * <li>The codec can decode to one of the specified color formats.</li> 175 * </ol> 176 * If multiple codecs are found, the one with the preferred color-format is taken. Color format 177 * preference is determined by the order of their appearance in the color format array. 178 * 179 * @param format The format the codec must decode. 180 * @param requiredColorFormats Array of target color spaces ordered by preference. 181 * @return A codec that meets the requirements, or null if no such codec was found. 182 */ 183 private static MediaCodec findDecoderCodec(MediaFormat format, int[] requiredColorFormats) { 184 TreeMap<Integer, String> candidateCodecs = new TreeMap<Integer, String>(); 185 SparseIntArray colorPriorities = intArrayToPriorityMap(requiredColorFormats); 186 for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { 187 // Get next codec 188 MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i); 189 190 // Check that this is a decoder 191 if (info.isEncoder()) { 192 continue; 193 } 194 195 // Check if this codec can decode the video in question 196 String requiredType = format.getString(MediaFormat.KEY_MIME); 197 String[] supportedTypes = info.getSupportedTypes(); 198 Set<String> typeSet = new HashSet<String>(Arrays.asList(supportedTypes)); 199 200 // Check if it can decode to one of the required color formats 201 if (typeSet.contains(requiredType)) { 202 CodecCapabilities capabilities = info.getCapabilitiesForType(requiredType); 203 for (int supportedColorFormat : capabilities.colorFormats) { 204 if (colorPriorities.indexOfKey(supportedColorFormat) >= 0) { 205 int priority = colorPriorities.get(supportedColorFormat); 206 candidateCodecs.put(priority, info.getName()); 207 } 208 } 209 } 210 } 211 212 // Pick the best codec (with the highest color priority) 213 if (candidateCodecs.isEmpty()) { 214 return null; 215 } else { 216 String bestCodec = candidateCodecs.firstEntry().getValue(); 217 return MediaCodec.createByCodecName(bestCodec); 218 } 219 } 220 221 private static SparseIntArray intArrayToPriorityMap(int[] values) { 222 SparseIntArray result = new SparseIntArray(); 223 for (int priority = 0; priority < values.length; ++priority) { 224 result.append(values[priority], priority); 225 } 226 return result; 227 } 228 229 private static void convertImage( 230 ByteBuffer input, ByteBuffer output, int colorFormat, int width, int height) { 231 switch (colorFormat) { 232 case CodecCapabilities.COLOR_Format32bitARGB8888: 233 ColorSpace.convertArgb8888ToRgba8888(input, output, width, height); 234 break; 235 case CodecCapabilities.COLOR_FormatYUV420Planar: 236 ColorSpace.convertYuv420pToRgba8888(input, output, width, height); 237 break; 238 default: 239 throw new RuntimeException("Unsupported color format: " + colorFormat + "!"); 240 } 241 } 242 243 } 244
