1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 package org.chromium.chromoting; 6 7 import android.app.ActionBar; 8 import android.app.Activity; 9 import android.graphics.Bitmap; 10 import android.graphics.Canvas; 11 import android.graphics.Color; 12 import android.graphics.Matrix; 13 import android.graphics.Paint; 14 import android.os.Bundle; 15 import android.os.Looper; 16 import android.text.InputType; 17 import android.util.Log; 18 import android.view.GestureDetector; 19 import android.view.MotionEvent; 20 import android.view.ScaleGestureDetector; 21 import android.view.SurfaceHolder; 22 import android.view.SurfaceView; 23 import android.view.inputmethod.EditorInfo; 24 import android.view.inputmethod.InputConnection; 25 26 import org.chromium.chromoting.jni.JniInterface; 27 28 /** 29 * The user interface for viewing and interacting with a specific remote host. 30 * It provides a canvas onto which the video feed is rendered, handles 31 * multitouch pan and zoom gestures, and collects and forwards input events. 32 */ 33 /** GUI element that holds the drawing canvas. */ 34 public class DesktopView extends SurfaceView implements Runnable, SurfaceHolder.Callback { 35 /** 36 * *Square* of the minimum displacement (in pixels) to be recognized as a scroll gesture. 37 * Setting this to a lower value forces more frequent canvas redraws during scrolling. 38 */ 39 private static final int MIN_SCROLL_DISTANCE = 8 * 8; 40 41 /** 42 * Minimum change to the scaling factor to be recognized as a zoom gesture. Setting lower 43 * values here will result in more frequent canvas redraws during zooming. 44 */ 45 private static final double MIN_ZOOM_FACTOR = 0.05; 46 47 /* 48 * These constants must match those in the generated struct protoc::MouseEvent_MouseButton. 49 */ 50 private static final int BUTTON_UNDEFINED = 0; 51 private static final int BUTTON_LEFT = 1; 52 private static final int BUTTON_RIGHT = 3; 53 54 /** Specifies one dimension of an image. */ 55 private static enum Constraint { 56 UNDEFINED, WIDTH, HEIGHT 57 } 58 59 private ActionBar mActionBar; 60 61 private GestureDetector mScroller; 62 private ScaleGestureDetector mZoomer; 63 64 /** Stores pan and zoom configuration and converts image coordinates to screen coordinates. */ 65 private Matrix mTransform; 66 67 private int mScreenWidth; 68 private int mScreenHeight; 69 70 /** Specifies the dimension by which the zoom level is being lower-bounded. */ 71 private Constraint mConstraint; 72 73 /** Whether the dimension of constraint should be reckecked on the next aspect ratio change. */ 74 private boolean mRecheckConstraint; 75 76 /** Whether the right edge of the image was visible on-screen during the last render. */ 77 private boolean mRightUsedToBeOut; 78 79 /** Whether the bottom edge of the image was visible on-screen during the last render. */ 80 private boolean mBottomUsedToBeOut; 81 82 private int mMouseButton; 83 private boolean mMousePressed; 84 85 public DesktopView(Activity context) { 86 super(context); 87 88 // Give this view keyboard focus, allowing us to customize the soft keyboard's settings. 89 setFocusableInTouchMode(true); 90 91 mActionBar = context.getActionBar(); 92 93 getHolder().addCallback(this); 94 DesktopListener listener = new DesktopListener(); 95 mScroller = new GestureDetector(context, listener, null, false); 96 mZoomer = new ScaleGestureDetector(context, listener); 97 98 mTransform = new Matrix(); 99 mScreenWidth = 0; 100 mScreenHeight = 0; 101 102 mConstraint = Constraint.UNDEFINED; 103 mRecheckConstraint = false; 104 105 mRightUsedToBeOut = false; 106 mBottomUsedToBeOut = false; 107 108 mMouseButton = BUTTON_UNDEFINED; 109 mMousePressed = false; 110 } 111 112 /** 113 * Redraws the canvas. This should be done on a non-UI thread or it could 114 * cause the UI to lag. Specifically, it is currently invoked on the native 115 * graphics thread using a JNI. 116 */ 117 @Override 118 public void run() { 119 if (Looper.myLooper() == Looper.getMainLooper()) { 120 Log.w("deskview", "Canvas being redrawn on UI thread"); 121 } 122 123 Bitmap image = JniInterface.retrieveVideoFrame(); 124 Canvas canvas = getHolder().lockCanvas(); 125 synchronized (mTransform) { 126 canvas.setMatrix(mTransform); 127 128 // Internal parameters of the transformation matrix. 129 float[] values = new float[9]; 130 mTransform.getValues(values); 131 132 // Screen coordinates of two defining points of the image. 133 float[] topleft = {0, 0}; 134 mTransform.mapPoints(topleft); 135 float[] bottomright = {image.getWidth(), image.getHeight()}; 136 mTransform.mapPoints(bottomright); 137 138 // Whether to rescale and recenter the view. 139 boolean recenter = false; 140 141 if (mConstraint == Constraint.UNDEFINED) { 142 mConstraint = (double)image.getWidth()/image.getHeight() > 143 (double)mScreenWidth/mScreenHeight ? Constraint.WIDTH : Constraint.HEIGHT; 144 recenter = true; // We always rescale and recenter after a rotation. 145 } 146 147 if (mConstraint == Constraint.WIDTH && 148 ((int)(bottomright[0] - topleft[0] + 0.5) < mScreenWidth || recenter)) { 149 // The vertical edges of the image are flush against the device's screen edges 150 // when the entire host screen is visible, and the user has zoomed out too far. 151 float imageMiddle = (float)image.getHeight() / 2; 152 float screenMiddle = (float)mScreenHeight / 2; 153 mTransform.setPolyToPoly( 154 new float[] {0, imageMiddle, image.getWidth(), imageMiddle}, 0, 155 new float[] {0, screenMiddle, mScreenWidth, screenMiddle}, 0, 2); 156 } else if (mConstraint == Constraint.HEIGHT && 157 ((int)(bottomright[1] - topleft[1] + 0.5) < mScreenHeight || recenter)) { 158 // The horizontal image edges are flush against the device's screen edges when 159 // the entire host screen is visible, and the user has zoomed out too far. 160 float imageCenter = (float)image.getWidth() / 2; 161 float screenCenter = (float)mScreenWidth / 2; 162 mTransform.setPolyToPoly( 163 new float[] {imageCenter, 0, imageCenter, image.getHeight()}, 0, 164 new float[] {screenCenter, 0, screenCenter, mScreenHeight}, 0, 2); 165 } else { 166 // It's fine for both members of a pair of image edges to be within the screen 167 // edges (or "out of bounds"); that simply means that the image is zoomed out as 168 // far as permissible. And both members of a pair can obviously be outside the 169 // screen's edges, which indicates that the image is zoomed in to far to see the 170 // whole host screen. However, if only one of a pair of edges has entered the 171 // screen, the user is attempting to scroll into a blank area of the canvas. 172 173 // A value of true means the corresponding edge has entered the screen's borders. 174 boolean leftEdgeOutOfBounds = values[Matrix.MTRANS_X] > 0; 175 boolean topEdgeOutOfBounds = values[Matrix.MTRANS_Y] > 0; 176 boolean rightEdgeOutOfBounds = bottomright[0] < mScreenWidth; 177 boolean bottomEdgeOutOfBounds = bottomright[1] < mScreenHeight; 178 179 // Prevent the user from scrolling past the left or right edge of the image. 180 if (leftEdgeOutOfBounds != rightEdgeOutOfBounds) { 181 if (leftEdgeOutOfBounds != mRightUsedToBeOut) { 182 // Make the left edge of the image flush with the left screen edge. 183 values[Matrix.MTRANS_X] = 0; 184 } 185 else { 186 // Make the right edge of the image flush with the right screen edge. 187 values[Matrix.MTRANS_X] += mScreenWidth - bottomright[0]; 188 } 189 } else { 190 // The else prevents this from being updated during the repositioning process, 191 // in which case the view would begin to oscillate. 192 mRightUsedToBeOut = rightEdgeOutOfBounds; 193 } 194 195 // Prevent the user from scrolling past the top or bottom edge of the image. 196 if (topEdgeOutOfBounds != bottomEdgeOutOfBounds) { 197 if (topEdgeOutOfBounds != mBottomUsedToBeOut) { 198 // Make the top edge of the image flush with the top screen edge. 199 values[Matrix.MTRANS_Y] = 0; 200 } else { 201 // Make the bottom edge of the image flush with the bottom screen edge. 202 values[Matrix.MTRANS_Y] += mScreenHeight - bottomright[1]; 203 } 204 } 205 else { 206 // The else prevents this from being updated during the repositioning process, 207 // in which case the view would begin to oscillate. 208 mBottomUsedToBeOut = bottomEdgeOutOfBounds; 209 } 210 211 mTransform.setValues(values); 212 } 213 214 canvas.setMatrix(mTransform); 215 } 216 217 canvas.drawColor(Color.BLACK); 218 canvas.drawBitmap(image, 0, 0, new Paint()); 219 getHolder().unlockCanvasAndPost(canvas); 220 } 221 222 /** 223 * Causes the next canvas redraw to perform a check for which screen dimension more tightly 224 * constrains the view of the image. This should be called between the time that a screen size 225 * change is requested and the time it actually occurs. If it is not called in such a case, the 226 * screen will not be rearranged as aggressively (which is desirable when the software keyboard 227 * appears in order to allow it to cover the image without forcing a resize). 228 */ 229 public void requestRecheckConstrainingDimension() { 230 mRecheckConstraint = true; 231 } 232 233 /** 234 * Called after the canvas is initially created, then after every 235 * subsequent resize, as when the display is rotated. 236 */ 237 @Override 238 public void surfaceChanged( 239 SurfaceHolder holder, int format, int width, int height) { 240 mActionBar.hide(); 241 242 synchronized (mTransform) { 243 mScreenWidth = width; 244 mScreenHeight = height; 245 246 if (mRecheckConstraint) { 247 mConstraint = Constraint.UNDEFINED; 248 mRecheckConstraint = false; 249 } 250 } 251 252 if (!JniInterface.redrawGraphics()) { 253 JniInterface.provideRedrawCallback(this); 254 } 255 } 256 257 /** Called when the canvas is first created. */ 258 @Override 259 public void surfaceCreated(SurfaceHolder holder) { 260 Log.i("deskview", "DesktopView.surfaceCreated(...)"); 261 } 262 263 /** 264 * Called when the canvas is finally destroyed. Marks the canvas as needing a redraw so that it 265 * will not be blank if the user later switches back to our window. 266 */ 267 @Override 268 public void surfaceDestroyed(SurfaceHolder holder) { 269 Log.i("deskview", "DesktopView.surfaceDestroyed(...)"); 270 271 // Stop this canvas from being redrawn. 272 JniInterface.provideRedrawCallback(null); 273 } 274 275 /** Called when a software keyboard is requested, and specifies its options. */ 276 @Override 277 public InputConnection onCreateInputConnection(EditorInfo outAttrs) { 278 // Disables rich input support and instead requests simple key events. 279 outAttrs.inputType = InputType.TYPE_NULL; 280 281 // Prevents most third-party IMEs from ignoring our Activity's adjustResize preference. 282 outAttrs.imeOptions |= EditorInfo.IME_FLAG_NO_FULLSCREEN; 283 284 // Ensures that keyboards will not decide to hide the remote desktop on small displays. 285 outAttrs.imeOptions |= EditorInfo.IME_FLAG_NO_EXTRACT_UI; 286 287 // Stops software keyboards from closing as soon as the enter key is pressed. 288 outAttrs.imeOptions |= EditorInfo.IME_MASK_ACTION | EditorInfo.IME_FLAG_NO_ENTER_ACTION; 289 290 return null; 291 } 292 293 /** Called when a mouse action is made. */ 294 private void handleMouseMovement(float x, float y, int button, boolean pressed) { 295 float[] coordinates = {x, y}; 296 297 // Coordinates are relative to the canvas, but we need image coordinates. 298 Matrix canvasToImage = new Matrix(); 299 mTransform.invert(canvasToImage); 300 canvasToImage.mapPoints(coordinates); 301 302 // Coordinates are now relative to the image, so transmit them to the host. 303 JniInterface.mouseAction((int)coordinates[0], (int)coordinates[1], button, pressed); 304 } 305 306 /** 307 * Called whenever the user attempts to touch the canvas. Forwards such 308 * events to the appropriate gesture detector until one accepts them. 309 */ 310 @Override 311 public boolean onTouchEvent(MotionEvent event) { 312 if (event.getPointerCount() == 3) { 313 mActionBar.show(); 314 } 315 316 boolean handled = mScroller.onTouchEvent(event) || mZoomer.onTouchEvent(event); 317 318 if (event.getPointerCount() == 1) { 319 float x = event.getRawX(); 320 float y = event.getY(); 321 322 switch (event.getActionMasked()) { 323 case MotionEvent.ACTION_DOWN: 324 Log.i("mouse", "Found a finger"); 325 mMouseButton = BUTTON_UNDEFINED; 326 mMousePressed = false; 327 break; 328 329 case MotionEvent.ACTION_MOVE: 330 Log.i("mouse", "Finger is dragging"); 331 if (mMouseButton == BUTTON_UNDEFINED) { 332 Log.i("mouse", "\tStarting left click"); 333 mMouseButton = BUTTON_LEFT; 334 mMousePressed = true; 335 } 336 break; 337 338 case MotionEvent.ACTION_UP: 339 Log.i("mouse", "Lost the finger"); 340 if (mMouseButton == BUTTON_UNDEFINED) { 341 // The user pressed and released without moving: do left click and release. 342 Log.i("mouse", "\tStarting and finishing left click"); 343 handleMouseMovement(x, y, BUTTON_LEFT, true); 344 mMouseButton = BUTTON_LEFT; 345 mMousePressed = false; 346 } 347 else if (mMousePressed) { 348 Log.i("mouse", "\tReleasing the currently-pressed button"); 349 mMousePressed = false; 350 } 351 else { 352 Log.w("mouse", "Button already in released state before gesture ended"); 353 } 354 break; 355 356 default: 357 return handled; 358 } 359 handleMouseMovement(x, y, mMouseButton, mMousePressed); 360 361 return true; 362 } 363 364 return handled; 365 } 366 367 /** Responds to touch events filtered by the gesture detectors. */ 368 private class DesktopListener extends GestureDetector.SimpleOnGestureListener 369 implements ScaleGestureDetector.OnScaleGestureListener { 370 /** 371 * Called when the user is scrolling. We refuse to accept or process the event unless it 372 * is being performed with 2 or more touch points, in order to reserve single-point touch 373 * events for emulating mouse input. 374 */ 375 @Override 376 public boolean onScroll( 377 MotionEvent e1, MotionEvent e2, float distanceX, float distanceY) { 378 if (e2.getPointerCount() < 2 || 379 Math.pow(distanceX, 2) + Math.pow(distanceY, 2) < MIN_SCROLL_DISTANCE) { 380 return false; 381 } 382 383 synchronized (mTransform) { 384 mTransform.postTranslate(-distanceX, -distanceY); 385 } 386 JniInterface.redrawGraphics(); 387 return true; 388 } 389 390 /** Called when the user is in the process of pinch-zooming. */ 391 @Override 392 public boolean onScale(ScaleGestureDetector detector) { 393 if (Math.abs(detector.getScaleFactor() - 1) < MIN_ZOOM_FACTOR) { 394 return false; 395 } 396 397 synchronized (mTransform) { 398 float scaleFactor = detector.getScaleFactor(); 399 mTransform.postScale( 400 scaleFactor, scaleFactor, detector.getFocusX(), detector.getFocusY()); 401 } 402 JniInterface.redrawGraphics(); 403 return true; 404 } 405 406 /** Called whenever a gesture starts. Always accepts the gesture so it isn't ignored. */ 407 @Override 408 public boolean onDown(MotionEvent e) { 409 return true; 410 } 411 412 /** 413 * Called when the user starts to zoom. Always accepts the zoom so that 414 * onScale() can decide whether to respond to it. 415 */ 416 @Override 417 public boolean onScaleBegin(ScaleGestureDetector detector) { 418 return true; 419 } 420 421 /** Called when the user is done zooming. Defers to onScale()'s judgement. */ 422 @Override 423 public void onScaleEnd(ScaleGestureDetector detector) { 424 onScale(detector); 425 } 426 427 /** Called when the user holds down on the screen. Starts a right-click. */ 428 @Override 429 public void onLongPress(MotionEvent e) { 430 if (e.getPointerCount() > 1) { 431 return; 432 } 433 434 float x = e.getRawX(); 435 float y = e.getY(); 436 437 Log.i("mouse", "Finger held down"); 438 if (mMousePressed) { 439 Log.i("mouse", "\tReleasing the currently-pressed button"); 440 handleMouseMovement(x, y, mMouseButton, false); 441 } 442 443 Log.i("mouse", "\tStarting right click"); 444 mMouseButton = BUTTON_RIGHT; 445 mMousePressed = true; 446 handleMouseMovement(x, y, mMouseButton, mMousePressed); 447 } 448 } 449 } 450