1 /* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package androidx.media.filterpacks.image; 18 19 import androidx.media.filterfw.Filter; 20 import androidx.media.filterfw.Frame; 21 import androidx.media.filterfw.FrameImage2D; 22 import androidx.media.filterfw.FrameType; 23 import androidx.media.filterfw.ImageShader; 24 import androidx.media.filterfw.MffContext; 25 import androidx.media.filterfw.OutputPort; 26 import androidx.media.filterfw.Signature; 27 28 import java.nio.ByteBuffer; 29 30 public class SobelFilter extends Filter { 31 32 private static final String mGradientXSource = 33 "precision mediump float;\n" 34 + "uniform sampler2D tex_sampler_0;\n" 35 + "uniform vec2 pix;\n" 36 + "varying vec2 v_texcoord;\n" 37 + "void main() {\n" 38 + " vec4 a1 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, -pix.y));\n" 39 + " vec4 a2 = -2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, 0.0));\n" 40 + " vec4 a3 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, +pix.y));\n" 41 + " vec4 b1 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, -pix.y));\n" 42 + " vec4 b2 = +2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, 0.0));\n" 43 + " vec4 b3 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, +pix.y));\n" 44 + " gl_FragColor = 0.5 + (a1 + a2 + a3 + b1 + b2 + b3) / 8.0;\n" 45 + "}\n"; 46 47 private static final String mGradientYSource = 48 "precision mediump float;\n" 49 + "uniform sampler2D tex_sampler_0;\n" 50 + "uniform vec2 pix;\n" 51 + "varying vec2 v_texcoord;\n" 52 + "void main() {\n" 53 + " vec4 a1 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, -pix.y));\n" 54 + " vec4 a2 = -2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(0.0, -pix.y));\n" 55 + " vec4 a3 = -1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, -pix.y));\n" 56 + " vec4 b1 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(-pix.x, +pix.y));\n" 57 + " vec4 b2 = +2.0 * texture2D(tex_sampler_0, v_texcoord + vec2(0.0, +pix.y));\n" 58 + " vec4 b3 = +1.0 * texture2D(tex_sampler_0, v_texcoord + vec2(+pix.x, +pix.y));\n" 59 + " gl_FragColor = 0.5 + (a1 + a2 + a3 + b1 + b2 + b3) / 8.0;\n" 60 + "}\n"; 61 62 private static final String mMagnitudeSource = 63 "precision mediump float;\n" 64 + "uniform sampler2D tex_sampler_0;\n" 65 + "uniform sampler2D tex_sampler_1;\n" 66 + "varying vec2 v_texcoord;\n" 67 + "void main() {\n" 68 + " vec4 gx = 2.0 * texture2D(tex_sampler_0, v_texcoord) - 1.0;\n" 69 + " vec4 gy = 2.0 * texture2D(tex_sampler_1, v_texcoord) - 1.0;\n" 70 + " gl_FragColor = vec4(sqrt(gx.rgb * gx.rgb + gy.rgb * gy.rgb), 1.0);\n" 71 + "}\n"; 72 73 private static final String mDirectionSource = 74 "precision mediump float;\n" 75 + "uniform sampler2D tex_sampler_0;\n" 76 + "uniform sampler2D tex_sampler_1;\n" 77 + "varying vec2 v_texcoord;\n" 78 + "void main() {\n" 79 + " vec4 gy = 2.0 * texture2D(tex_sampler_1, v_texcoord) - 1.0;\n" 80 + " vec4 gx = 2.0 * texture2D(tex_sampler_0, v_texcoord) - 1.0;\n" 81 + " gl_FragColor = vec4((atan(gy.rgb, gx.rgb) + 3.14) / (2.0 * 3.14), 1.0);\n" 82 + "}\n"; 83 84 private ImageShader mGradientXShader; 85 private ImageShader mGradientYShader; 86 private ImageShader mMagnitudeShader; 87 private ImageShader mDirectionShader; 88 89 private FrameType mImageType; 90 91 public SobelFilter(MffContext context, String name) { 92 super(context, name); 93 } 94 95 @Override 96 public Signature getSignature() { 97 // TODO: we will address the issue of READ_GPU / WRITE_GPU when using CPU filters later. 98 FrameType imageIn = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU); 99 FrameType imageOut = FrameType.image2D(FrameType.ELEMENT_RGBA8888, FrameType.WRITE_GPU); 100 return new Signature().addInputPort("image", Signature.PORT_REQUIRED, imageIn) 101 .addOutputPort("direction", Signature.PORT_OPTIONAL, imageOut) 102 .addOutputPort("magnitude", Signature.PORT_OPTIONAL, imageOut).disallowOtherPorts(); 103 } 104 105 @Override 106 protected void onPrepare() { 107 if (isOpenGLSupported()) { 108 mGradientXShader = new ImageShader(mGradientXSource); 109 mGradientYShader = new ImageShader(mGradientYSource); 110 mMagnitudeShader = new ImageShader(mMagnitudeSource); 111 mDirectionShader = new ImageShader(mDirectionSource); 112 mImageType = FrameType.image2D( 113 FrameType.ELEMENT_RGBA8888, FrameType.READ_GPU | FrameType.WRITE_GPU); 114 } 115 } 116 117 @Override 118 protected void onProcess() { 119 OutputPort magnitudePort = getConnectedOutputPort("magnitude"); 120 OutputPort directionPort = getConnectedOutputPort("direction"); 121 FrameImage2D inputImage = getConnectedInputPort("image").pullFrame().asFrameImage2D(); 122 int[] inputDims = inputImage.getDimensions(); 123 124 FrameImage2D magImage = (magnitudePort != null) ? 125 magnitudePort.fetchAvailableFrame(inputDims).asFrameImage2D() : null; 126 FrameImage2D dirImage = (directionPort != null) ? 127 directionPort.fetchAvailableFrame(inputDims).asFrameImage2D() : null; 128 if (isOpenGLSupported()) { 129 FrameImage2D gxFrame = Frame.create(mImageType, inputDims).asFrameImage2D(); 130 FrameImage2D gyFrame = Frame.create(mImageType, inputDims).asFrameImage2D(); 131 mGradientXShader.setUniformValue("pix", new float[] {1f/inputDims[0], 1f/inputDims[1]}); 132 mGradientYShader.setUniformValue("pix", new float[] {1f/inputDims[0], 1f/inputDims[1]}); 133 mGradientXShader.process(inputImage, gxFrame); 134 mGradientYShader.process(inputImage, gyFrame); 135 FrameImage2D[] gradientFrames = new FrameImage2D[] { gxFrame, gyFrame }; 136 if (magnitudePort != null) { 137 mMagnitudeShader.processMulti(gradientFrames, magImage); 138 } 139 if (directionPort != null) { 140 mDirectionShader.processMulti(gradientFrames, dirImage); 141 } 142 gxFrame.release(); 143 gyFrame.release(); 144 } else { 145 ByteBuffer inputBuffer = inputImage.lockBytes(Frame.MODE_READ); 146 ByteBuffer magBuffer = (magImage != null) ? 147 magImage.lockBytes(Frame.MODE_WRITE) : null; 148 ByteBuffer dirBuffer = (dirImage != null) ? 149 dirImage.lockBytes(Frame.MODE_WRITE) : null; 150 sobelOperator(inputImage.getWidth(), inputImage.getHeight(), 151 inputBuffer, magBuffer, dirBuffer); 152 inputImage.unlock(); 153 if (magImage != null) { 154 magImage.unlock(); 155 } 156 if (dirImage != null) { 157 dirImage.unlock(); 158 } 159 } 160 if (magImage != null) { 161 magnitudePort.pushFrame(magImage); 162 } 163 if (dirImage != null) { 164 directionPort.pushFrame(dirImage); 165 } 166 } 167 168 private static native boolean sobelOperator(int width, int height, 169 ByteBuffer imageBuffer, ByteBuffer magBuffer, ByteBuffer dirBudder); 170 171 static { 172 System.loadLibrary("smartcamera_jni"); 173 } 174 } 175
