1 /* 2 ** Copyright 2015, The Android Open Source Project 3 ** 4 ** Licensed under the Apache License, Version 2.0 (the "License"); 5 ** you may not use this file except in compliance with the License. 6 ** You may obtain a copy of the License at 7 ** 8 ** http://www.apache.org/licenses/LICENSE-2.0 9 ** 10 ** Unless required by applicable law or agreed to in writing, software 11 ** distributed under the License is distributed on an "AS IS" BASIS, 12 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 ** See the License for the specific language governing permissions and 14 ** limitations under the License. 15 */ 16 17 package com.android.server.accessibility; 18 19 import android.content.Context; 20 import android.gesture.Gesture; 21 import android.gesture.GestureLibraries; 22 import android.gesture.GestureLibrary; 23 import android.gesture.GesturePoint; 24 import android.gesture.GestureStore; 25 import android.gesture.GestureStroke; 26 import android.gesture.Prediction; 27 import android.util.Slog; 28 import android.util.TypedValue; 29 import android.view.GestureDetector; 30 import android.view.MotionEvent; 31 import android.view.VelocityTracker; 32 import android.view.ViewConfiguration; 33 34 import com.android.internal.R; 35 36 import java.util.ArrayList; 37 38 /** 39 * This class handles gesture detection for the Touch Explorer. It collects 40 * touch events and determines when they match a gesture, as well as when they 41 * won't match a gesture. These state changes are then surfaced to mListener. 42 */ 43 class AccessibilityGestureDetector extends GestureDetector.SimpleOnGestureListener { 44 45 private static final boolean DEBUG = false; 46 47 // Tag for logging received events. 48 private static final String LOG_TAG = "AccessibilityGestureDetector"; 49 50 /** 51 * Listener functions are called as a result of onMoveEvent(). The current 52 * MotionEvent in the context of these functions is the event passed into 53 * onMotionEvent. 54 */ 55 public interface Listener { 56 /** 57 * Called when the user has performed a double tap and then held down 58 * the second tap. 59 * 60 * @param event The most recent MotionEvent received. 61 * @param policyFlags The policy flags of the most recent event. 62 */ 63 void onDoubleTapAndHold(MotionEvent event, int policyFlags); 64 65 /** 66 * Called when the user lifts their finger on the second tap of a double 67 * tap. 68 * 69 * @param event The most recent MotionEvent received. 70 * @param policyFlags The policy flags of the most recent event. 71 * 72 * @return true if the event is consumed, else false 73 */ 74 boolean onDoubleTap(MotionEvent event, int policyFlags); 75 76 /** 77 * Called when the system has decided the event stream is a gesture. 78 * 79 * @return true if the event is consumed, else false 80 */ 81 boolean onGestureStarted(); 82 83 /** 84 * Called when an event stream is recognized as a gesture. 85 * 86 * @param gestureId ID of the gesture that was recognized. 87 * 88 * @return true if the event is consumed, else false 89 */ 90 boolean onGestureCompleted(int gestureId); 91 92 /** 93 * Called when the system has decided an event stream doesn't match any 94 * known gesture. 95 * 96 * @param event The most recent MotionEvent received. 97 * @param policyFlags The policy flags of the most recent event. 98 * 99 * @return true if the event is consumed, else false 100 */ 101 public boolean onGestureCancelled(MotionEvent event, int policyFlags); 102 } 103 104 private final Listener mListener; 105 private final GestureDetector mGestureDetector; 106 107 // The library for gesture detection. 108 private final GestureLibrary mGestureLibrary; 109 110 // Indicates that a single tap has occurred. 111 private boolean mFirstTapDetected; 112 113 // Indicates that the down event of a double tap has occured. 114 private boolean mDoubleTapDetected; 115 116 // Indicates that motion events are being collected to match a gesture. 117 private boolean mRecognizingGesture; 118 119 // Indicates that we've collected enough data to be sure it could be a 120 // gesture. 121 private boolean mGestureStarted; 122 123 // Indicates that motion events from the second pointer are being checked 124 // for a double tap. 125 private boolean mSecondFingerDoubleTap; 126 127 // Tracks the most recent time where ACTION_POINTER_DOWN was sent for the 128 // second pointer. 129 private long mSecondPointerDownTime; 130 131 // Policy flags of the previous event. 132 private int mPolicyFlags; 133 134 // These values track the previous point that was saved to use for gesture 135 // detection. They are only updated when the user moves more than the 136 // recognition threshold. 137 private float mPreviousGestureX; 138 private float mPreviousGestureY; 139 140 // These values track the previous point that was used to determine if there 141 // was a transition into or out of gesture detection. They are updated when 142 // the user moves more than the detection threshold. 143 private float mBaseX; 144 private float mBaseY; 145 private long mBaseTime; 146 147 // This is the calculated movement threshold used track if the user is still 148 // moving their finger. 149 private final float mGestureDetectionThreshold; 150 151 // Buffer for storing points for gesture detection. 152 private final ArrayList<GesturePoint> mStrokeBuffer = new ArrayList<GesturePoint>(100); 153 154 // The minimal delta between moves to add a gesture point. 155 private static final int TOUCH_TOLERANCE = 3; 156 157 // The minimal score for accepting a predicted gesture. 158 private static final float MIN_PREDICTION_SCORE = 2.0f; 159 160 // Distance a finger must travel before we decide if it is a gesture or not. 161 private static final int GESTURE_CONFIRM_MM = 10; 162 163 // Time threshold used to determine if an interaction is a gesture or not. 164 // If the first movement of 1cm takes longer than this value, we assume it's 165 // a slow movement, and therefore not a gesture. 166 // 167 // This value was determined by measuring the time for the first 1cm 168 // movement when gesturing, and touch exploring. Based on user testing, 169 // all gestures started with the initial movement taking less than 100ms. 170 // When touch exploring, the first movement almost always takes longer than 171 // 200ms. From this data, 200ms seems the best value to decide what 172 // kind of interaction it is. 173 private static final long CANCEL_ON_PAUSE_THRESHOLD_NOT_STARTED_MS = 200; 174 175 // Time threshold used to determine if a gesture should be cancelled. If 176 // the finger pauses for longer than this delay, the ongoing gesture is 177 // cancelled. 178 private static final long CANCEL_ON_PAUSE_THRESHOLD_STARTED_MS = 500; 179 180 AccessibilityGestureDetector(Context context, Listener listener) { 181 mListener = listener; 182 183 mGestureDetector = new GestureDetector(context, this); 184 mGestureDetector.setOnDoubleTapListener(this); 185 186 mGestureLibrary = GestureLibraries.fromRawResource(context, R.raw.accessibility_gestures); 187 mGestureLibrary.setOrientationStyle(8 /* GestureStore.ORIENTATION_SENSITIVE_8 */); 188 mGestureLibrary.setSequenceType(GestureStore.SEQUENCE_SENSITIVE); 189 mGestureLibrary.load(); 190 191 mGestureDetectionThreshold = TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_MM, 1, 192 context.getResources().getDisplayMetrics()) * GESTURE_CONFIRM_MM; 193 } 194 195 /** 196 * Handle a motion event. If an action is completed, the appropriate 197 * callback on mListener is called, and the return value of the callback is 198 * passed to the caller. 199 * 200 * @param event The raw motion event. It's important that this be the raw 201 * event, before any transformations have been applied, so that measurements 202 * can be made in physical units. 203 * @param policyFlags Policy flags for the event. 204 * 205 * @return true if the event is consumed, else false 206 */ 207 public boolean onMotionEvent(MotionEvent event, int policyFlags) { 208 final float x = event.getX(); 209 final float y = event.getY(); 210 final long time = event.getEventTime(); 211 212 mPolicyFlags = policyFlags; 213 switch (event.getActionMasked()) { 214 case MotionEvent.ACTION_DOWN: 215 mDoubleTapDetected = false; 216 mSecondFingerDoubleTap = false; 217 mRecognizingGesture = true; 218 mGestureStarted = false; 219 mPreviousGestureX = x; 220 mPreviousGestureY = y; 221 mStrokeBuffer.clear(); 222 mStrokeBuffer.add(new GesturePoint(x, y, time)); 223 224 mBaseX = x; 225 mBaseY = y; 226 mBaseTime = time; 227 break; 228 229 case MotionEvent.ACTION_MOVE: 230 if (mRecognizingGesture) { 231 final float deltaX = mBaseX - x; 232 final float deltaY = mBaseY - y; 233 final double moveDelta = Math.hypot(deltaX, deltaY); 234 if (moveDelta > mGestureDetectionThreshold) { 235 // If the pointer has moved more than the threshold, 236 // update the stored values. 237 mBaseX = x; 238 mBaseY = y; 239 mBaseTime = time; 240 241 // Since the pointer has moved, this is not a double 242 // tap. 243 mFirstTapDetected = false; 244 mDoubleTapDetected = false; 245 246 // If this hasn't been confirmed as a gesture yet, send 247 // the event. 248 if (!mGestureStarted) { 249 mGestureStarted = true; 250 return mListener.onGestureStarted(); 251 } 252 } else if (!mFirstTapDetected) { 253 // The finger may not move if they are double tapping. 254 // In that case, we shouldn't cancel the gesture. 255 final long timeDelta = time - mBaseTime; 256 final long threshold = mGestureStarted ? 257 CANCEL_ON_PAUSE_THRESHOLD_STARTED_MS : 258 CANCEL_ON_PAUSE_THRESHOLD_NOT_STARTED_MS; 259 260 // If the pointer hasn't moved for longer than the 261 // timeout, cancel gesture detection. 262 if (timeDelta > threshold) { 263 cancelGesture(); 264 return mListener.onGestureCancelled(event, policyFlags); 265 } 266 } 267 268 final float dX = Math.abs(x - mPreviousGestureX); 269 final float dY = Math.abs(y - mPreviousGestureY); 270 if (dX >= TOUCH_TOLERANCE || dY >= TOUCH_TOLERANCE) { 271 mPreviousGestureX = x; 272 mPreviousGestureY = y; 273 mStrokeBuffer.add(new GesturePoint(x, y, time)); 274 } 275 } 276 break; 277 278 case MotionEvent.ACTION_UP: 279 if (mDoubleTapDetected) { 280 return finishDoubleTap(event, policyFlags); 281 } 282 if (mGestureStarted) { 283 mStrokeBuffer.add(new GesturePoint(x, y, time)); 284 285 return recognizeGesture(event, policyFlags); 286 } 287 break; 288 289 case MotionEvent.ACTION_POINTER_DOWN: 290 // Once a second finger is used, we're definitely not 291 // recognizing a gesture. 292 cancelGesture(); 293 294 if (event.getPointerCount() == 2) { 295 // If this was the second finger, attempt to recognize double 296 // taps on it. 297 mSecondFingerDoubleTap = true; 298 mSecondPointerDownTime = time; 299 } else { 300 // If there are more than two fingers down, stop watching 301 // for a double tap. 302 mSecondFingerDoubleTap = false; 303 } 304 break; 305 306 case MotionEvent.ACTION_POINTER_UP: 307 // If we're detecting taps on the second finger, see if we 308 // should finish the double tap. 309 if (mSecondFingerDoubleTap && mDoubleTapDetected) { 310 return finishDoubleTap(event, policyFlags); 311 } 312 break; 313 314 case MotionEvent.ACTION_CANCEL: 315 clear(); 316 break; 317 } 318 319 // If we're detecting taps on the second finger, map events from the 320 // finger to the first finger. 321 if (mSecondFingerDoubleTap) { 322 MotionEvent newEvent = mapSecondPointerToFirstPointer(event); 323 if (newEvent == null) { 324 return false; 325 } 326 boolean handled = mGestureDetector.onTouchEvent(newEvent); 327 newEvent.recycle(); 328 return handled; 329 } 330 331 if (!mRecognizingGesture) { 332 return false; 333 } 334 335 // Pass the event on to the standard gesture detector. 336 return mGestureDetector.onTouchEvent(event); 337 } 338 339 public void clear() { 340 mFirstTapDetected = false; 341 mDoubleTapDetected = false; 342 mSecondFingerDoubleTap = false; 343 mGestureStarted = false; 344 mGestureDetector.onTouchEvent(MotionEvent.obtain(0L, 0L, MotionEvent.ACTION_CANCEL, 345 0.0f, 0.0f, 0)); 346 cancelGesture(); 347 } 348 349 public boolean firstTapDetected() { 350 return mFirstTapDetected; 351 } 352 353 @Override 354 public void onLongPress(MotionEvent e) { 355 maybeSendLongPress(e, mPolicyFlags); 356 } 357 358 @Override 359 public boolean onSingleTapUp(MotionEvent event) { 360 mFirstTapDetected = true; 361 return false; 362 } 363 364 @Override 365 public boolean onSingleTapConfirmed(MotionEvent event) { 366 clear(); 367 return false; 368 } 369 370 @Override 371 public boolean onDoubleTap(MotionEvent event) { 372 // The processing of the double tap is deferred until the finger is 373 // lifted, so that we can detect a long press on the second tap. 374 mDoubleTapDetected = true; 375 return false; 376 } 377 378 private void maybeSendLongPress(MotionEvent event, int policyFlags) { 379 if (!mDoubleTapDetected) { 380 return; 381 } 382 383 clear(); 384 385 mListener.onDoubleTapAndHold(event, policyFlags); 386 } 387 388 private boolean finishDoubleTap(MotionEvent event, int policyFlags) { 389 clear(); 390 391 return mListener.onDoubleTap(event, policyFlags); 392 } 393 394 private void cancelGesture() { 395 mRecognizingGesture = false; 396 mGestureStarted = false; 397 mStrokeBuffer.clear(); 398 } 399 400 private boolean recognizeGesture(MotionEvent event, int policyFlags) { 401 Gesture gesture = new Gesture(); 402 gesture.addStroke(new GestureStroke(mStrokeBuffer)); 403 404 ArrayList<Prediction> predictions = mGestureLibrary.recognize(gesture); 405 if (!predictions.isEmpty()) { 406 Prediction bestPrediction = predictions.get(0); 407 if (bestPrediction.score >= MIN_PREDICTION_SCORE) { 408 if (DEBUG) { 409 Slog.i(LOG_TAG, "gesture: " + bestPrediction.name + " score: " 410 + bestPrediction.score); 411 } 412 try { 413 final int gestureId = Integer.parseInt(bestPrediction.name); 414 return mListener.onGestureCompleted(gestureId); 415 } catch (NumberFormatException nfe) { 416 Slog.w(LOG_TAG, "Non numeric gesture id:" + bestPrediction.name); 417 } 418 } 419 } 420 421 return mListener.onGestureCancelled(event, policyFlags); 422 } 423 424 private MotionEvent mapSecondPointerToFirstPointer(MotionEvent event) { 425 // Only map basic events when two fingers are down. 426 if (event.getPointerCount() != 2 || 427 (event.getActionMasked() != MotionEvent.ACTION_POINTER_DOWN && 428 event.getActionMasked() != MotionEvent.ACTION_POINTER_UP && 429 event.getActionMasked() != MotionEvent.ACTION_MOVE)) { 430 return null; 431 } 432 433 int action = event.getActionMasked(); 434 435 if (action == MotionEvent.ACTION_POINTER_DOWN) { 436 action = MotionEvent.ACTION_DOWN; 437 } else if (action == MotionEvent.ACTION_POINTER_UP) { 438 action = MotionEvent.ACTION_UP; 439 } 440 441 // Map the information from the second pointer to the first. 442 return MotionEvent.obtain(mSecondPointerDownTime, event.getEventTime(), action, 443 event.getX(1), event.getY(1), event.getPressure(1), event.getSize(1), 444 event.getMetaState(), event.getXPrecision(), event.getYPrecision(), 445 event.getDeviceId(), event.getEdgeFlags()); 446 } 447 } 448