1 /* 2 * libjingle 3 * Copyright 2013, Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 package org.appspot.apprtc; 29 30 import android.content.Context; 31 import android.graphics.Point; 32 import android.graphics.Rect; 33 import android.opengl.GLES20; 34 import android.opengl.GLSurfaceView; 35 import android.util.Log; 36 37 import org.webrtc.VideoRenderer.I420Frame; 38 39 import java.nio.ByteBuffer; 40 import java.nio.ByteOrder; 41 import java.nio.FloatBuffer; 42 import java.util.EnumMap; 43 44 import javax.microedition.khronos.egl.EGLConfig; 45 import javax.microedition.khronos.opengles.GL10; 46 47 /** 48 * A GLSurfaceView{,.Renderer} that efficiently renders YUV frames from local & 49 * remote VideoTracks using the GPU for CSC. Clients will want to call the 50 * constructor, setSize() and updateFrame() as appropriate, but none of the 51 * other public methods of this class are of interest to clients (only to system 52 * classes). 53 */ 54 public class VideoStreamsView 55 extends GLSurfaceView 56 implements GLSurfaceView.Renderer { 57 58 /** Identify which of the two video streams is being addressed. */ 59 public static enum Endpoint { LOCAL, REMOTE }; 60 61 private final static String TAG = "VideoStreamsView"; 62 private EnumMap<Endpoint, Rect> rects = 63 new EnumMap<Endpoint, Rect>(Endpoint.class); 64 private Point screenDimensions; 65 // [0] are local Y,U,V, [1] are remote Y,U,V. 66 private int[][] yuvTextures = { { -1, -1, -1}, {-1, -1, -1 }}; 67 private int posLocation = -1; 68 private long lastFPSLogTime = System.nanoTime(); 69 private long numFramesSinceLastLog = 0; 70 private FramePool framePool = new FramePool(); 71 // Accessed on multiple threads! Must be synchronized. 72 private EnumMap<Endpoint, I420Frame> framesToRender = 73 new EnumMap<Endpoint, I420Frame>(Endpoint.class); 74 75 public VideoStreamsView(Context c, Point screenDimensions) { 76 super(c); 77 this.screenDimensions = screenDimensions; 78 setEGLContextClientVersion(2); 79 setRenderer(this); 80 setRenderMode(RENDERMODE_WHEN_DIRTY); 81 } 82 83 /** Queue |frame| to be uploaded. */ 84 public void queueFrame(final Endpoint stream, I420Frame frame) { 85 // Paying for the copy of the YUV data here allows CSC and painting time 86 // to get spent on the render thread instead of the UI thread. 87 abortUnless(framePool.validateDimensions(frame), "Frame too large!"); 88 final I420Frame frameCopy = framePool.takeFrame(frame).copyFrom(frame); 89 boolean needToScheduleRender; 90 synchronized (framesToRender) { 91 // A new render needs to be scheduled (via updateFrames()) iff there isn't 92 // already a render scheduled, which is true iff framesToRender is empty. 93 needToScheduleRender = framesToRender.isEmpty(); 94 I420Frame frameToDrop = framesToRender.put(stream, frameCopy); 95 if (frameToDrop != null) { 96 framePool.returnFrame(frameToDrop); 97 } 98 } 99 if (needToScheduleRender) { 100 queueEvent(new Runnable() { 101 public void run() { 102 updateFrames(); 103 } 104 }); 105 } 106 } 107 108 // Upload the planes from |framesToRender| to the textures owned by this View. 109 private void updateFrames() { 110 I420Frame localFrame = null; 111 I420Frame remoteFrame = null; 112 synchronized (framesToRender) { 113 localFrame = framesToRender.remove(Endpoint.LOCAL); 114 remoteFrame = framesToRender.remove(Endpoint.REMOTE); 115 } 116 if (localFrame != null) { 117 texImage2D(localFrame, yuvTextures[0]); 118 framePool.returnFrame(localFrame); 119 } 120 if (remoteFrame != null) { 121 texImage2D(remoteFrame, yuvTextures[1]); 122 framePool.returnFrame(remoteFrame); 123 } 124 abortUnless(localFrame != null || remoteFrame != null, 125 "Nothing to render!"); 126 requestRender(); 127 } 128 129 /** Inform this View of the dimensions of frames coming from |stream|. */ 130 public void setSize(Endpoint stream, int width, int height) { 131 // Generate 3 texture ids for Y/U/V and place them into |textures|, 132 // allocating enough storage for |width|x|height| pixels. 133 int[] textures = yuvTextures[stream == Endpoint.LOCAL ? 0 : 1]; 134 GLES20.glGenTextures(3, textures, 0); 135 for (int i = 0; i < 3; ++i) { 136 int w = i == 0 ? width : width / 2; 137 int h = i == 0 ? height : height / 2; 138 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); 139 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[i]); 140 GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0, 141 GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, null); 142 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 143 GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); 144 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 145 GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); 146 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 147 GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); 148 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, 149 GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); 150 } 151 checkNoGLES2Error(); 152 } 153 154 @Override 155 protected void onMeasure(int unusedX, int unusedY) { 156 // Go big or go home! 157 setMeasuredDimension(screenDimensions.x, screenDimensions.y); 158 } 159 160 @Override 161 public void onSurfaceChanged(GL10 unused, int width, int height) { 162 GLES20.glViewport(0, 0, width, height); 163 checkNoGLES2Error(); 164 } 165 166 @Override 167 public void onDrawFrame(GL10 unused) { 168 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 169 drawRectangle(yuvTextures[1], remoteVertices); 170 drawRectangle(yuvTextures[0], localVertices); 171 ++numFramesSinceLastLog; 172 long now = System.nanoTime(); 173 if (lastFPSLogTime == -1 || now - lastFPSLogTime > 1e9) { 174 double fps = numFramesSinceLastLog / ((now - lastFPSLogTime) / 1e9); 175 Log.d(TAG, "Rendered FPS: " + fps); 176 lastFPSLogTime = now; 177 numFramesSinceLastLog = 1; 178 } 179 checkNoGLES2Error(); 180 } 181 182 @Override 183 public void onSurfaceCreated(GL10 unused, EGLConfig config) { 184 int program = GLES20.glCreateProgram(); 185 addShaderTo(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER_STRING, program); 186 addShaderTo(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER_STRING, program); 187 188 GLES20.glLinkProgram(program); 189 int[] result = new int[] { GLES20.GL_FALSE }; 190 result[0] = GLES20.GL_FALSE; 191 GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, result, 0); 192 abortUnless(result[0] == GLES20.GL_TRUE, 193 GLES20.glGetProgramInfoLog(program)); 194 GLES20.glUseProgram(program); 195 196 GLES20.glUniform1i(GLES20.glGetUniformLocation(program, "y_tex"), 0); 197 GLES20.glUniform1i(GLES20.glGetUniformLocation(program, "u_tex"), 1); 198 GLES20.glUniform1i(GLES20.glGetUniformLocation(program, "v_tex"), 2); 199 200 // Actually set in drawRectangle(), but queried only once here. 201 posLocation = GLES20.glGetAttribLocation(program, "in_pos"); 202 203 int tcLocation = GLES20.glGetAttribLocation(program, "in_tc"); 204 GLES20.glEnableVertexAttribArray(tcLocation); 205 GLES20.glVertexAttribPointer( 206 tcLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords); 207 208 GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f); 209 checkNoGLES2Error(); 210 } 211 212 // Wrap a float[] in a direct FloatBuffer using native byte order. 213 private static FloatBuffer directNativeFloatBuffer(float[] array) { 214 FloatBuffer buffer = ByteBuffer.allocateDirect(array.length * 4).order( 215 ByteOrder.nativeOrder()).asFloatBuffer(); 216 buffer.put(array); 217 buffer.flip(); 218 return buffer; 219 } 220 221 // Upload the YUV planes from |frame| to |textures|. 222 private void texImage2D(I420Frame frame, int[] textures) { 223 for (int i = 0; i < 3; ++i) { 224 ByteBuffer plane = frame.yuvPlanes[i]; 225 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); 226 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[i]); 227 int w = i == 0 ? frame.width : frame.width / 2; 228 int h = i == 0 ? frame.height : frame.height / 2; 229 abortUnless(w == frame.yuvStrides[i], frame.yuvStrides[i] + "!=" + w); 230 GLES20.glTexImage2D( 231 GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0, 232 GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, plane); 233 } 234 checkNoGLES2Error(); 235 } 236 237 // Draw |textures| using |vertices| (X,Y coordinates). 238 private void drawRectangle(int[] textures, FloatBuffer vertices) { 239 for (int i = 0; i < 3; ++i) { 240 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); 241 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[i]); 242 } 243 244 GLES20.glVertexAttribPointer( 245 posLocation, 2, GLES20.GL_FLOAT, false, 0, vertices); 246 GLES20.glEnableVertexAttribArray(posLocation); 247 248 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); 249 checkNoGLES2Error(); 250 } 251 252 // Compile & attach a |type| shader specified by |source| to |program|. 253 private static void addShaderTo( 254 int type, String source, int program) { 255 int[] result = new int[] { GLES20.GL_FALSE }; 256 int shader = GLES20.glCreateShader(type); 257 GLES20.glShaderSource(shader, source); 258 GLES20.glCompileShader(shader); 259 GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0); 260 abortUnless(result[0] == GLES20.GL_TRUE, 261 GLES20.glGetShaderInfoLog(shader) + ", source: " + source); 262 GLES20.glAttachShader(program, shader); 263 GLES20.glDeleteShader(shader); 264 checkNoGLES2Error(); 265 } 266 267 // Poor-man's assert(): die with |msg| unless |condition| is true. 268 private static void abortUnless(boolean condition, String msg) { 269 if (!condition) { 270 throw new RuntimeException(msg); 271 } 272 } 273 274 // Assert that no OpenGL ES 2.0 error has been raised. 275 private static void checkNoGLES2Error() { 276 int error = GLES20.glGetError(); 277 abortUnless(error == GLES20.GL_NO_ERROR, "GLES20 error: " + error); 278 } 279 280 // Remote image should span the full screen. 281 private static final FloatBuffer remoteVertices = directNativeFloatBuffer( 282 new float[] { -1, 1, -1, -1, 1, 1, 1, -1 }); 283 284 // Local image should be thumbnailish. 285 private static final FloatBuffer localVertices = directNativeFloatBuffer( 286 new float[] { 0.6f, 0.9f, 0.6f, 0.6f, 0.9f, 0.9f, 0.9f, 0.6f }); 287 288 // Texture Coordinates mapping the entire texture. 289 private static final FloatBuffer textureCoords = directNativeFloatBuffer( 290 new float[] { 0, 0, 0, 1, 1, 0, 1, 1 }); 291 292 // Pass-through vertex shader. 293 private static final String VERTEX_SHADER_STRING = 294 "varying vec2 interp_tc;\n" + 295 "\n" + 296 "attribute vec4 in_pos;\n" + 297 "attribute vec2 in_tc;\n" + 298 "\n" + 299 "void main() {\n" + 300 " gl_Position = in_pos;\n" + 301 " interp_tc = in_tc;\n" + 302 "}\n"; 303 304 // YUV to RGB pixel shader. Loads a pixel from each plane and pass through the 305 // matrix. 306 private static final String FRAGMENT_SHADER_STRING = 307 "precision mediump float;\n" + 308 "varying vec2 interp_tc;\n" + 309 "\n" + 310 "uniform sampler2D y_tex;\n" + 311 "uniform sampler2D u_tex;\n" + 312 "uniform sampler2D v_tex;\n" + 313 "\n" + 314 "void main() {\n" + 315 " float y = texture2D(y_tex, interp_tc).r;\n" + 316 " float u = texture2D(u_tex, interp_tc).r - .5;\n" + 317 " float v = texture2D(v_tex, interp_tc).r - .5;\n" + 318 // CSC according to http://www.fourcc.org/fccyvrgb.php 319 " gl_FragColor = vec4(y + 1.403 * v, " + 320 " y - 0.344 * u - 0.714 * v, " + 321 " y + 1.77 * u, 1);\n" + 322 "}\n"; 323 } 324