1 /* 2 ** Copyright 2015, The Android Open Source Project 3 ** 4 ** Licensed under the Apache License, Version 2.0 (the "License"); 5 ** you may not use this file except in compliance with the License. 6 ** You may obtain a copy of the License at 7 ** 8 ** http://www.apache.org/licenses/LICENSE-2.0 9 ** 10 ** Unless required by applicable law or agreed to in writing, software 11 ** distributed under the License is distributed on an "AS IS" BASIS, 12 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 ** See the License for the specific language governing permissions and 14 ** limitations under the License. 15 */ 16 17 package com.android.server.accessibility; 18 19 import android.accessibilityservice.AccessibilityService; 20 import android.content.Context; 21 import android.gesture.Gesture; 22 import android.gesture.GesturePoint; 23 import android.gesture.GestureStore; 24 import android.gesture.GestureStroke; 25 import android.gesture.Prediction; 26 import android.graphics.PointF; 27 import android.util.Slog; 28 import android.util.TypedValue; 29 import android.view.GestureDetector; 30 import android.view.MotionEvent; 31 import android.view.ViewConfiguration; 32 33 import com.android.internal.R; 34 35 import java.util.ArrayList; 36 37 /** 38 * This class handles gesture detection for the Touch Explorer. It collects 39 * touch events and determines when they match a gesture, as well as when they 40 * won't match a gesture. These state changes are then surfaced to mListener. 41 */ 42 class AccessibilityGestureDetector extends GestureDetector.SimpleOnGestureListener { 43 44 private static final boolean DEBUG = false; 45 46 // Tag for logging received events. 47 private static final String LOG_TAG = "AccessibilityGestureDetector"; 48 49 // Constants for sampling motion event points. 50 // We sample based on a minimum distance between points, primarily to improve accuracy by 51 // reducing noisy minor changes in direction. 52 private static final float MIN_INCHES_BETWEEN_SAMPLES = 0.1f; 53 private final float mMinPixelsBetweenSamplesX; 54 private final float mMinPixelsBetweenSamplesY; 55 56 // Constants for separating gesture segments 57 private static final float ANGLE_THRESHOLD = 0.0f; 58 59 // Constants for line segment directions 60 private static final int LEFT = 0; 61 private static final int RIGHT = 1; 62 private static final int UP = 2; 63 private static final int DOWN = 3; 64 private static final int[][] DIRECTIONS_TO_GESTURE_ID = { 65 { 66 AccessibilityService.GESTURE_SWIPE_LEFT, 67 AccessibilityService.GESTURE_SWIPE_LEFT_AND_RIGHT, 68 AccessibilityService.GESTURE_SWIPE_LEFT_AND_UP, 69 AccessibilityService.GESTURE_SWIPE_LEFT_AND_DOWN 70 }, 71 { 72 AccessibilityService.GESTURE_SWIPE_RIGHT_AND_LEFT, 73 AccessibilityService.GESTURE_SWIPE_RIGHT, 74 AccessibilityService.GESTURE_SWIPE_RIGHT_AND_UP, 75 AccessibilityService.GESTURE_SWIPE_RIGHT_AND_DOWN 76 }, 77 { 78 AccessibilityService.GESTURE_SWIPE_UP_AND_LEFT, 79 AccessibilityService.GESTURE_SWIPE_UP_AND_RIGHT, 80 AccessibilityService.GESTURE_SWIPE_UP, 81 AccessibilityService.GESTURE_SWIPE_UP_AND_DOWN 82 }, 83 { 84 AccessibilityService.GESTURE_SWIPE_DOWN_AND_LEFT, 85 AccessibilityService.GESTURE_SWIPE_DOWN_AND_RIGHT, 86 AccessibilityService.GESTURE_SWIPE_DOWN_AND_UP, 87 AccessibilityService.GESTURE_SWIPE_DOWN 88 } 89 }; 90 91 92 /** 93 * Listener functions are called as a result of onMoveEvent(). The current 94 * MotionEvent in the context of these functions is the event passed into 95 * onMotionEvent. 96 */ 97 public interface Listener { 98 /** 99 * Called when the user has performed a double tap and then held down 100 * the second tap. 101 * 102 * @param event The most recent MotionEvent received. 103 * @param policyFlags The policy flags of the most recent event. 104 */ 105 void onDoubleTapAndHold(MotionEvent event, int policyFlags); 106 107 /** 108 * Called when the user lifts their finger on the second tap of a double 109 * tap. 110 * 111 * @param event The most recent MotionEvent received. 112 * @param policyFlags The policy flags of the most recent event. 113 * 114 * @return true if the event is consumed, else false 115 */ 116 boolean onDoubleTap(MotionEvent event, int policyFlags); 117 118 /** 119 * Called when the system has decided the event stream is a gesture. 120 * 121 * @return true if the event is consumed, else false 122 */ 123 boolean onGestureStarted(); 124 125 /** 126 * Called when an event stream is recognized as a gesture. 127 * 128 * @param gestureId ID of the gesture that was recognized. 129 * 130 * @return true if the event is consumed, else false 131 */ 132 boolean onGestureCompleted(int gestureId); 133 134 /** 135 * Called when the system has decided an event stream doesn't match any 136 * known gesture. 137 * 138 * @param event The most recent MotionEvent received. 139 * @param policyFlags The policy flags of the most recent event. 140 * 141 * @return true if the event is consumed, else false 142 */ 143 public boolean onGestureCancelled(MotionEvent event, int policyFlags); 144 } 145 146 private final Listener mListener; 147 private final Context mContext; // Retained for on-demand construction of GestureDetector. 148 protected GestureDetector mGestureDetector; // Double-tap detector. Visible for test. 149 150 // Indicates that a single tap has occurred. 151 private boolean mFirstTapDetected; 152 153 // Indicates that the down event of a double tap has occured. 154 private boolean mDoubleTapDetected; 155 156 // Indicates that motion events are being collected to match a gesture. 157 private boolean mRecognizingGesture; 158 159 // Indicates that we've collected enough data to be sure it could be a 160 // gesture. 161 private boolean mGestureStarted; 162 163 // Indicates that motion events from the second pointer are being checked 164 // for a double tap. 165 private boolean mSecondFingerDoubleTap; 166 167 // Tracks the most recent time where ACTION_POINTER_DOWN was sent for the 168 // second pointer. 169 private long mSecondPointerDownTime; 170 171 // Policy flags of the previous event. 172 private int mPolicyFlags; 173 174 // These values track the previous point that was saved to use for gesture 175 // detection. They are only updated when the user moves more than the 176 // recognition threshold. 177 private float mPreviousGestureX; 178 private float mPreviousGestureY; 179 180 // These values track the previous point that was used to determine if there 181 // was a transition into or out of gesture detection. They are updated when 182 // the user moves more than the detection threshold. 183 private float mBaseX; 184 private float mBaseY; 185 private long mBaseTime; 186 187 // This is the calculated movement threshold used track if the user is still 188 // moving their finger. 189 private final float mGestureDetectionThreshold; 190 191 // Buffer for storing points for gesture detection. 192 private final ArrayList<GesturePoint> mStrokeBuffer = new ArrayList<GesturePoint>(100); 193 194 // The minimal delta between moves to add a gesture point. 195 private static final int TOUCH_TOLERANCE = 3; 196 197 // The minimal score for accepting a predicted gesture. 198 private static final float MIN_PREDICTION_SCORE = 2.0f; 199 200 // Distance a finger must travel before we decide if it is a gesture or not. 201 private static final int GESTURE_CONFIRM_MM = 10; 202 203 // Time threshold used to determine if an interaction is a gesture or not. 204 // If the first movement of 1cm takes longer than this value, we assume it's 205 // a slow movement, and therefore not a gesture. 206 // 207 // This value was determined by measuring the time for the first 1cm 208 // movement when gesturing, and touch exploring. Based on user testing, 209 // all gestures started with the initial movement taking less than 100ms. 210 // When touch exploring, the first movement almost always takes longer than 211 // 200ms. 212 private static final long CANCEL_ON_PAUSE_THRESHOLD_NOT_STARTED_MS = 150; 213 214 // Time threshold used to determine if a gesture should be cancelled. If 215 // the finger takes more than this time to move 1cm, the ongoing gesture is 216 // cancelled. 217 private static final long CANCEL_ON_PAUSE_THRESHOLD_STARTED_MS = 300; 218 219 AccessibilityGestureDetector(Context context, Listener listener) { 220 mListener = listener; 221 mContext = context; 222 223 mGestureDetectionThreshold = TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_MM, 1, 224 context.getResources().getDisplayMetrics()) * GESTURE_CONFIRM_MM; 225 226 // Calculate minimum gesture velocity 227 final float pixelsPerInchX = context.getResources().getDisplayMetrics().xdpi; 228 final float pixelsPerInchY = context.getResources().getDisplayMetrics().ydpi; 229 mMinPixelsBetweenSamplesX = MIN_INCHES_BETWEEN_SAMPLES * pixelsPerInchX; 230 mMinPixelsBetweenSamplesY = MIN_INCHES_BETWEEN_SAMPLES * pixelsPerInchY; 231 } 232 233 /** 234 * Handle a motion event. If an action is completed, the appropriate 235 * callback on mListener is called, and the return value of the callback is 236 * passed to the caller. 237 * 238 * @param event The raw motion event. It's important that this be the raw 239 * event, before any transformations have been applied, so that measurements 240 * can be made in physical units. 241 * @param policyFlags Policy flags for the event. 242 * 243 * @return true if the event is consumed, else false 244 */ 245 public boolean onMotionEvent(MotionEvent event, int policyFlags) { 246 247 // Construct GestureDetector double-tap detector on demand, so that testable sub-class 248 // can use mock GestureDetector. 249 // TODO: Break the circular dependency between GestureDetector's constructor and 250 // AccessibilityGestureDetector's constructor. Construct GestureDetector in TouchExplorer, 251 // using a GestureDetector listener owned by TouchExplorer, which passes double-tap state 252 // information to AccessibilityGestureDetector. 253 if (mGestureDetector == null) { 254 mGestureDetector = new GestureDetector(mContext, this); 255 mGestureDetector.setOnDoubleTapListener(this); 256 } 257 258 final float x = event.getX(); 259 final float y = event.getY(); 260 final long time = event.getEventTime(); 261 262 mPolicyFlags = policyFlags; 263 switch (event.getActionMasked()) { 264 case MotionEvent.ACTION_DOWN: 265 mDoubleTapDetected = false; 266 mSecondFingerDoubleTap = false; 267 mRecognizingGesture = true; 268 mGestureStarted = false; 269 mPreviousGestureX = x; 270 mPreviousGestureY = y; 271 mStrokeBuffer.clear(); 272 mStrokeBuffer.add(new GesturePoint(x, y, time)); 273 274 mBaseX = x; 275 mBaseY = y; 276 mBaseTime = time; 277 break; 278 279 case MotionEvent.ACTION_MOVE: 280 if (mRecognizingGesture) { 281 final float deltaX = mBaseX - x; 282 final float deltaY = mBaseY - y; 283 final double moveDelta = Math.hypot(deltaX, deltaY); 284 if (moveDelta > mGestureDetectionThreshold) { 285 // If the pointer has moved more than the threshold, 286 // update the stored values. 287 mBaseX = x; 288 mBaseY = y; 289 mBaseTime = time; 290 291 // Since the pointer has moved, this is not a double 292 // tap. 293 mFirstTapDetected = false; 294 mDoubleTapDetected = false; 295 296 // If this hasn't been confirmed as a gesture yet, send 297 // the event. 298 if (!mGestureStarted) { 299 mGestureStarted = true; 300 return mListener.onGestureStarted(); 301 } 302 } else if (!mFirstTapDetected) { 303 // The finger may not move if they are double tapping. 304 // In that case, we shouldn't cancel the gesture. 305 final long timeDelta = time - mBaseTime; 306 final long threshold = mGestureStarted ? 307 CANCEL_ON_PAUSE_THRESHOLD_STARTED_MS : 308 CANCEL_ON_PAUSE_THRESHOLD_NOT_STARTED_MS; 309 310 // If the pointer hasn't moved for longer than the 311 // timeout, cancel gesture detection. 312 if (timeDelta > threshold) { 313 cancelGesture(); 314 return mListener.onGestureCancelled(event, policyFlags); 315 } 316 } 317 318 final float dX = Math.abs(x - mPreviousGestureX); 319 final float dY = Math.abs(y - mPreviousGestureY); 320 if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { 321 mPreviousGestureX = x; 322 mPreviousGestureY = y; 323 mStrokeBuffer.add(new GesturePoint(x, y, time)); 324 } 325 } 326 break; 327 328 case MotionEvent.ACTION_UP: 329 if (mDoubleTapDetected) { 330 return finishDoubleTap(event, policyFlags); 331 } 332 if (mGestureStarted) { 333 final float dX = Math.abs(x - mPreviousGestureX); 334 final float dY = Math.abs(y - mPreviousGestureY); 335 if (dX >= mMinPixelsBetweenSamplesX || dY >= mMinPixelsBetweenSamplesY) { 336 mStrokeBuffer.add(new GesturePoint(x, y, time)); 337 } 338 return recognizeGesture(event, policyFlags); 339 } 340 break; 341 342 case MotionEvent.ACTION_POINTER_DOWN: 343 // Once a second finger is used, we're definitely not 344 // recognizing a gesture. 345 cancelGesture(); 346 347 if (event.getPointerCount() == 2) { 348 // If this was the second finger, attempt to recognize double 349 // taps on it. 350 mSecondFingerDoubleTap = true; 351 mSecondPointerDownTime = time; 352 } else { 353 // If there are more than two fingers down, stop watching 354 // for a double tap. 355 mSecondFingerDoubleTap = false; 356 } 357 break; 358 359 case MotionEvent.ACTION_POINTER_UP: 360 // If we're detecting taps on the second finger, see if we 361 // should finish the double tap. 362 if (mSecondFingerDoubleTap && mDoubleTapDetected) { 363 return finishDoubleTap(event, policyFlags); 364 } 365 break; 366 367 case MotionEvent.ACTION_CANCEL: 368 clear(); 369 break; 370 } 371 372 // If we're detecting taps on the second finger, map events from the 373 // finger to the first finger. 374 if (mSecondFingerDoubleTap) { 375 MotionEvent newEvent = mapSecondPointerToFirstPointer(event); 376 if (newEvent == null) { 377 return false; 378 } 379 boolean handled = mGestureDetector.onTouchEvent(newEvent); 380 newEvent.recycle(); 381 return handled; 382 } 383 384 if (!mRecognizingGesture) { 385 return false; 386 } 387 388 // Pass the event on to the standard gesture detector. 389 return mGestureDetector.onTouchEvent(event); 390 } 391 392 public void clear() { 393 mFirstTapDetected = false; 394 mDoubleTapDetected = false; 395 mSecondFingerDoubleTap = false; 396 mGestureStarted = false; 397 mGestureDetector.onTouchEvent(MotionEvent.obtain(0L, 0L, MotionEvent.ACTION_CANCEL, 398 0.0f, 0.0f, 0)); 399 cancelGesture(); 400 } 401 402 public boolean firstTapDetected() { 403 return mFirstTapDetected; 404 } 405 406 @Override 407 public void onLongPress(MotionEvent e) { 408 maybeSendLongPress(e, mPolicyFlags); 409 } 410 411 @Override 412 public boolean onSingleTapUp(MotionEvent event) { 413 mFirstTapDetected = true; 414 return false; 415 } 416 417 @Override 418 public boolean onSingleTapConfirmed(MotionEvent event) { 419 clear(); 420 return false; 421 } 422 423 @Override 424 public boolean onDoubleTap(MotionEvent event) { 425 // The processing of the double tap is deferred until the finger is 426 // lifted, so that we can detect a long press on the second tap. 427 mDoubleTapDetected = true; 428 return false; 429 } 430 431 private void maybeSendLongPress(MotionEvent event, int policyFlags) { 432 if (!mDoubleTapDetected) { 433 return; 434 } 435 436 clear(); 437 438 mListener.onDoubleTapAndHold(event, policyFlags); 439 } 440 441 private boolean finishDoubleTap(MotionEvent event, int policyFlags) { 442 clear(); 443 444 return mListener.onDoubleTap(event, policyFlags); 445 } 446 447 private void cancelGesture() { 448 mRecognizingGesture = false; 449 mGestureStarted = false; 450 mStrokeBuffer.clear(); 451 } 452 453 /** 454 * Looks at the sequence of motions in mStrokeBuffer, classifies the gesture, then calls 455 * Listener callbacks for success or failure. 456 * 457 * @param event The raw motion event to pass to the listener callbacks. 458 * @param policyFlags Policy flags for the event. 459 * 460 * @return true if the event is consumed, else false 461 */ 462 private boolean recognizeGesture(MotionEvent event, int policyFlags) { 463 if (mStrokeBuffer.size() < 2) { 464 return mListener.onGestureCancelled(event, policyFlags); 465 } 466 467 // Look at mStrokeBuffer and extract 2 line segments, delimited by near-perpendicular 468 // direction change. 469 // Method: for each sampled motion event, check the angle of the most recent motion vector 470 // versus the preceding motion vector, and segment the line if the angle is about 471 // 90 degrees. 472 473 ArrayList<PointF> path = new ArrayList<>(); 474 PointF lastDelimiter = new PointF(mStrokeBuffer.get(0).x, mStrokeBuffer.get(0).y); 475 path.add(lastDelimiter); 476 477 float dX = 0; // Sum of unit vectors from last delimiter to each following point 478 float dY = 0; 479 int count = 0; // Number of points since last delimiter 480 float length = 0; // Vector length from delimiter to most recent point 481 482 PointF next = new PointF(); 483 for (int i = 1; i < mStrokeBuffer.size(); ++i) { 484 next = new PointF(mStrokeBuffer.get(i).x, mStrokeBuffer.get(i).y); 485 if (count > 0) { 486 // Average of unit vectors from delimiter to following points 487 float currentDX = dX / count; 488 float currentDY = dY / count; 489 490 // newDelimiter is a possible new delimiter, based on a vector with length from 491 // the last delimiter to the previous point, but in the direction of the average 492 // unit vector from delimiter to previous points. 493 // Using the averaged vector has the effect of "squaring off the curve", 494 // creating a sharper angle between the last motion and the preceding motion from 495 // the delimiter. In turn, this sharper angle achieves the splitting threshold 496 // even in a gentle curve. 497 PointF newDelimiter = new PointF(length * currentDX + lastDelimiter.x, 498 length * currentDY + lastDelimiter.y); 499 500 // Unit vector from newDelimiter to the most recent point 501 float nextDX = next.x - newDelimiter.x; 502 float nextDY = next.y - newDelimiter.y; 503 float nextLength = (float) Math.sqrt(nextDX * nextDX + nextDY * nextDY); 504 nextDX = nextDX / nextLength; 505 nextDY = nextDY / nextLength; 506 507 // Compare the initial motion direction to the most recent motion direction, 508 // and segment the line if direction has changed by about 90 degrees. 509 float dot = currentDX * nextDX + currentDY * nextDY; 510 if (dot < ANGLE_THRESHOLD) { 511 path.add(newDelimiter); 512 lastDelimiter = newDelimiter; 513 dX = 0; 514 dY = 0; 515 count = 0; 516 } 517 } 518 519 // Vector from last delimiter to most recent point 520 float currentDX = next.x - lastDelimiter.x; 521 float currentDY = next.y - lastDelimiter.y; 522 length = (float) Math.sqrt(currentDX * currentDX + currentDY * currentDY); 523 524 // Increment sum of unit vectors from delimiter to each following point 525 count = count + 1; 526 dX = dX + currentDX / length; 527 dY = dY + currentDY / length; 528 } 529 530 path.add(next); 531 Slog.i(LOG_TAG, "path=" + path.toString()); 532 533 // Classify line segments, and call Listener callbacks. 534 return recognizeGesturePath(event, policyFlags, path); 535 } 536 537 /** 538 * Classifies a pair of line segments, by direction. 539 * Calls Listener callbacks for success or failure. 540 * 541 * @param event The raw motion event to pass to the listener's onGestureCanceled method. 542 * @param policyFlags Policy flags for the event. 543 * @param path A sequence of motion line segments derived from motion points in mStrokeBuffer. 544 * 545 * @return true if the event is consumed, else false 546 */ 547 private boolean recognizeGesturePath(MotionEvent event, int policyFlags, 548 ArrayList<PointF> path) { 549 550 if (path.size() == 2) { 551 PointF start = path.get(0); 552 PointF end = path.get(1); 553 554 float dX = end.x - start.x; 555 float dY = end.y - start.y; 556 int direction = toDirection(dX, dY); 557 switch (direction) { 558 case LEFT: 559 return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_LEFT); 560 case RIGHT: 561 return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_RIGHT); 562 case UP: 563 return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_UP); 564 case DOWN: 565 return mListener.onGestureCompleted(AccessibilityService.GESTURE_SWIPE_DOWN); 566 default: 567 // Do nothing. 568 } 569 570 } else if (path.size() == 3) { 571 PointF start = path.get(0); 572 PointF mid = path.get(1); 573 PointF end = path.get(2); 574 575 float dX0 = mid.x - start.x; 576 float dY0 = mid.y - start.y; 577 578 float dX1 = end.x - mid.x; 579 float dY1 = end.y - mid.y; 580 581 int segmentDirection0 = toDirection(dX0, dY0); 582 int segmentDirection1 = toDirection(dX1, dY1); 583 int gestureId = DIRECTIONS_TO_GESTURE_ID[segmentDirection0][segmentDirection1]; 584 return mListener.onGestureCompleted(gestureId); 585 } 586 // else if (path.size() < 2 || 3 < path.size()) then no gesture recognized. 587 return mListener.onGestureCancelled(event, policyFlags); 588 } 589 590 /** Maps a vector to a dominant direction in set {LEFT, RIGHT, UP, DOWN}. */ 591 private static int toDirection(float dX, float dY) { 592 if (Math.abs(dX) > Math.abs(dY)) { 593 // Horizontal 594 return (dX < 0) ? LEFT : RIGHT; 595 } else { 596 // Vertical 597 return (dY < 0) ? UP : DOWN; 598 } 599 } 600 601 private MotionEvent mapSecondPointerToFirstPointer(MotionEvent event) { 602 // Only map basic events when two fingers are down. 603 if (event.getPointerCount() != 2 || 604 (event.getActionMasked() != MotionEvent.ACTION_POINTER_DOWN && 605 event.getActionMasked() != MotionEvent.ACTION_POINTER_UP && 606 event.getActionMasked() != MotionEvent.ACTION_MOVE)) { 607 return null; 608 } 609 610 int action = event.getActionMasked(); 611 612 if (action == MotionEvent.ACTION_POINTER_DOWN) { 613 action = MotionEvent.ACTION_DOWN; 614 } else if (action == MotionEvent.ACTION_POINTER_UP) { 615 action = MotionEvent.ACTION_UP; 616 } 617 618 // Map the information from the second pointer to the first. 619 return MotionEvent.obtain(mSecondPointerDownTime, event.getEventTime(), action, 620 event.getX(1), event.getY(1), event.getPressure(1), event.getSize(1), 621 event.getMetaState(), event.getXPrecision(), event.getYPrecision(), 622 event.getDeviceId(), event.getEdgeFlags()); 623 } 624 } 625