1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.accessibility; 18 19 import android.accessibilityservice.GestureDescription; 20 import android.accessibilityservice.GestureDescription.GestureStep; 21 import android.accessibilityservice.GestureDescription.TouchPoint; 22 import android.accessibilityservice.IAccessibilityServiceClient; 23 import android.os.Handler; 24 import android.os.Looper; 25 import android.os.Message; 26 import android.os.RemoteException; 27 import android.os.SystemClock; 28 import android.util.IntArray; 29 import android.util.Slog; 30 import android.util.SparseArray; 31 import android.util.SparseIntArray; 32 import android.view.InputDevice; 33 import android.view.KeyEvent; 34 import android.view.MotionEvent; 35 import android.view.WindowManagerPolicy; 36 import android.view.accessibility.AccessibilityEvent; 37 import com.android.internal.os.SomeArgs; 38 39 import java.util.ArrayList; 40 import java.util.List; 41 42 /** 43 * Injects MotionEvents to permit {@code AccessibilityService}s to touch the screen on behalf of 44 * users. 45 * <p> 46 * All methods except {@code injectEvents} must be called only from the main thread. 47 */ 48 public class MotionEventInjector implements EventStreamTransformation, Handler.Callback { 49 private static final String LOG_TAG = "MotionEventInjector"; 50 private static final int MESSAGE_SEND_MOTION_EVENT = 1; 51 private static final int MESSAGE_INJECT_EVENTS = 2; 52 53 /** 54 * Constants used to initialize all MotionEvents 55 */ 56 private static final int EVENT_META_STATE = 0; 57 private static final int EVENT_BUTTON_STATE = 0; 58 private static final int EVENT_DEVICE_ID = 0; 59 private static final int EVENT_EDGE_FLAGS = 0; 60 private static final int EVENT_SOURCE = InputDevice.SOURCE_TOUCHSCREEN; 61 private static final int EVENT_FLAGS = 0; 62 private static final float EVENT_X_PRECISION = 1; 63 private static final float EVENT_Y_PRECISION = 1; 64 65 private static MotionEvent.PointerCoords[] sPointerCoords; 66 private static MotionEvent.PointerProperties[] sPointerProps; 67 68 private final Handler mHandler; 69 private final SparseArray<Boolean> mOpenGesturesInProgress = new SparseArray<>(); 70 71 private EventStreamTransformation mNext; 72 private IAccessibilityServiceClient mServiceInterfaceForCurrentGesture; 73 private IntArray mSequencesInProgress = new IntArray(5); 74 private boolean mIsDestroyed = false; 75 private TouchPoint[] mLastTouchPoints; 76 private int mNumLastTouchPoints; 77 private long mDownTime; 78 private long mLastScheduledEventTime; 79 private SparseIntArray mStrokeIdToPointerId = new SparseIntArray(5); 80 81 /** 82 * @param looper A looper on the main thread to use for dispatching new events 83 */ 84 public MotionEventInjector(Looper looper) { 85 mHandler = new Handler(looper, this); 86 } 87 88 /** 89 * @param handler A handler to post messages. Exposes internal state for testing only. 90 */ 91 public MotionEventInjector(Handler handler) { 92 mHandler = handler; 93 } 94 95 /** 96 * Schedule a gesture for injection. The gesture is defined by a set of {@code GestureStep}s, 97 * from which {@code MotionEvent}s will be derived. All gestures currently in progress will be 98 * cancelled. 99 * 100 * @param gestureSteps The gesture steps to inject. 101 * @param serviceInterface The interface to call back with a result when the gesture is 102 * either complete or cancelled. 103 */ 104 public void injectEvents(List<GestureStep> gestureSteps, 105 IAccessibilityServiceClient serviceInterface, int sequence) { 106 SomeArgs args = SomeArgs.obtain(); 107 args.arg1 = gestureSteps; 108 args.arg2 = serviceInterface; 109 args.argi1 = sequence; 110 mHandler.sendMessage(mHandler.obtainMessage(MESSAGE_INJECT_EVENTS, args)); 111 } 112 113 @Override 114 public void onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { 115 cancelAnyPendingInjectedEvents(); 116 sendMotionEventToNext(event, rawEvent, policyFlags); 117 } 118 119 @Override 120 public void onKeyEvent(KeyEvent event, int policyFlags) { 121 if (mNext != null) { 122 mNext.onKeyEvent(event, policyFlags); 123 } 124 } 125 126 @Override 127 public void onAccessibilityEvent(AccessibilityEvent event) { 128 if (mNext != null) { 129 mNext.onAccessibilityEvent(event); 130 } 131 } 132 133 @Override 134 public void setNext(EventStreamTransformation next) { 135 mNext = next; 136 } 137 138 @Override 139 public void clearEvents(int inputSource) { 140 /* 141 * Reset state for motion events passing through so we won't send a cancel event for 142 * them. 143 */ 144 if (!mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { 145 mOpenGesturesInProgress.put(inputSource, false); 146 } 147 } 148 149 @Override 150 public void onDestroy() { 151 cancelAnyPendingInjectedEvents(); 152 mIsDestroyed = true; 153 } 154 155 @Override 156 public boolean handleMessage(Message message) { 157 if (message.what == MESSAGE_INJECT_EVENTS) { 158 SomeArgs args = (SomeArgs) message.obj; 159 injectEventsMainThread((List<GestureStep>) args.arg1, 160 (IAccessibilityServiceClient) args.arg2, args.argi1); 161 args.recycle(); 162 return true; 163 } 164 if (message.what != MESSAGE_SEND_MOTION_EVENT) { 165 Slog.e(LOG_TAG, "Unknown message: " + message.what); 166 return false; 167 } 168 MotionEvent motionEvent = (MotionEvent) message.obj; 169 sendMotionEventToNext(motionEvent, motionEvent, WindowManagerPolicy.FLAG_PASS_TO_USER); 170 boolean isEndOfSequence = message.arg1 != 0; 171 if (isEndOfSequence) { 172 notifyService(mServiceInterfaceForCurrentGesture, mSequencesInProgress.get(0), true); 173 mSequencesInProgress.remove(0); 174 } 175 return true; 176 } 177 178 private void injectEventsMainThread(List<GestureStep> gestureSteps, 179 IAccessibilityServiceClient serviceInterface, int sequence) { 180 if (mIsDestroyed) { 181 try { 182 serviceInterface.onPerformGestureResult(sequence, false); 183 } catch (RemoteException re) { 184 Slog.e(LOG_TAG, "Error sending status with mIsDestroyed to " + serviceInterface, 185 re); 186 } 187 return; 188 } 189 190 if (mNext == null) { 191 notifyService(serviceInterface, sequence, false); 192 return; 193 } 194 195 boolean continuingGesture = newGestureTriesToContinueOldOne(gestureSteps); 196 197 if (continuingGesture) { 198 if ((serviceInterface != mServiceInterfaceForCurrentGesture) 199 || !prepareToContinueOldGesture(gestureSteps)) { 200 cancelAnyPendingInjectedEvents(); 201 notifyService(serviceInterface, sequence, false); 202 return; 203 } 204 } 205 if (!continuingGesture) { 206 cancelAnyPendingInjectedEvents(); 207 // Injected gestures have been canceled, but real gestures still need cancelling 208 cancelAnyGestureInProgress(EVENT_SOURCE); 209 } 210 mServiceInterfaceForCurrentGesture = serviceInterface; 211 212 long currentTime = SystemClock.uptimeMillis(); 213 List<MotionEvent> events = getMotionEventsFromGestureSteps(gestureSteps, 214 (mSequencesInProgress.size() == 0) ? currentTime : mLastScheduledEventTime); 215 if (events.isEmpty()) { 216 notifyService(serviceInterface, sequence, false); 217 return; 218 } 219 mSequencesInProgress.add(sequence); 220 221 for (int i = 0; i < events.size(); i++) { 222 MotionEvent event = events.get(i); 223 int isEndOfSequence = (i == events.size() - 1) ? 1 : 0; 224 Message message = mHandler.obtainMessage( 225 MESSAGE_SEND_MOTION_EVENT, isEndOfSequence, 0, event); 226 mLastScheduledEventTime = event.getEventTime(); 227 mHandler.sendMessageDelayed(message, Math.max(0, event.getEventTime() - currentTime)); 228 } 229 } 230 231 private boolean newGestureTriesToContinueOldOne(List<GestureStep> gestureSteps) { 232 if (gestureSteps.isEmpty()) { 233 return false; 234 } 235 GestureStep firstStep = gestureSteps.get(0); 236 for (int i = 0; i < firstStep.numTouchPoints; i++) { 237 if (!firstStep.touchPoints[i].mIsStartOfPath) { 238 return true; 239 } 240 } 241 return false; 242 } 243 244 /** 245 * A gesture can only continue a gesture if it contains intermediate points that continue 246 * each continued stroke of the last gesture, and no extra points. 247 * 248 * @param gestureSteps The steps of the new gesture 249 * @return {@code true} if the new gesture could continue the last one dispatched. {@code false} 250 * otherwise. 251 */ 252 private boolean prepareToContinueOldGesture(List<GestureStep> gestureSteps) { 253 if (gestureSteps.isEmpty() || (mLastTouchPoints == null) || (mNumLastTouchPoints == 0)) { 254 return false; 255 } 256 GestureStep firstStep = gestureSteps.get(0); 257 // Make sure all of the continuing paths match up 258 int numContinuedStrokes = 0; 259 for (int i = 0; i < firstStep.numTouchPoints; i++) { 260 TouchPoint touchPoint = firstStep.touchPoints[i]; 261 if (!touchPoint.mIsStartOfPath) { 262 int continuedPointerId = mStrokeIdToPointerId 263 .get(touchPoint.mContinuedStrokeId, -1); 264 if (continuedPointerId == -1) { 265 return false; 266 } 267 mStrokeIdToPointerId.put(touchPoint.mStrokeId, continuedPointerId); 268 int lastPointIndex = findPointByStrokeId( 269 mLastTouchPoints, mNumLastTouchPoints, touchPoint.mContinuedStrokeId); 270 if (lastPointIndex < 0) { 271 return false; 272 } 273 if (mLastTouchPoints[lastPointIndex].mIsEndOfPath 274 || (mLastTouchPoints[lastPointIndex].mX != touchPoint.mX) 275 || (mLastTouchPoints[lastPointIndex].mY != touchPoint.mY)) { 276 return false; 277 } 278 // Update the last touch point to match the continuation, so the gestures will 279 // line up 280 mLastTouchPoints[lastPointIndex].mStrokeId = touchPoint.mStrokeId; 281 } 282 numContinuedStrokes++; 283 } 284 // Make sure we didn't miss any paths 285 for (int i = 0; i < mNumLastTouchPoints; i++) { 286 if (!mLastTouchPoints[i].mIsEndOfPath) { 287 numContinuedStrokes--; 288 } 289 } 290 return numContinuedStrokes == 0; 291 } 292 293 private void sendMotionEventToNext(MotionEvent event, MotionEvent rawEvent, 294 int policyFlags) { 295 if (mNext != null) { 296 mNext.onMotionEvent(event, rawEvent, policyFlags); 297 if (event.getActionMasked() == MotionEvent.ACTION_DOWN) { 298 mOpenGesturesInProgress.put(event.getSource(), true); 299 } 300 if ((event.getActionMasked() == MotionEvent.ACTION_UP) 301 || (event.getActionMasked() == MotionEvent.ACTION_CANCEL)) { 302 mOpenGesturesInProgress.put(event.getSource(), false); 303 } 304 } 305 } 306 307 private void cancelAnyGestureInProgress(int source) { 308 if ((mNext != null) && mOpenGesturesInProgress.get(source, false)) { 309 long now = SystemClock.uptimeMillis(); 310 MotionEvent cancelEvent = 311 obtainMotionEvent(now, now, MotionEvent.ACTION_CANCEL, getLastTouchPoints(), 1); 312 sendMotionEventToNext(cancelEvent, cancelEvent, 313 WindowManagerPolicy.FLAG_PASS_TO_USER); 314 mOpenGesturesInProgress.put(source, false); 315 } 316 } 317 318 private void cancelAnyPendingInjectedEvents() { 319 if (mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { 320 mHandler.removeMessages(MESSAGE_SEND_MOTION_EVENT); 321 cancelAnyGestureInProgress(EVENT_SOURCE); 322 for (int i = mSequencesInProgress.size() - 1; i >= 0; i--) { 323 notifyService(mServiceInterfaceForCurrentGesture, 324 mSequencesInProgress.get(i), false); 325 mSequencesInProgress.remove(i); 326 } 327 } else if (mNumLastTouchPoints != 0) { 328 // An injected gesture is in progress and waiting for a continuation. Cancel it. 329 cancelAnyGestureInProgress(EVENT_SOURCE); 330 } 331 mNumLastTouchPoints = 0; 332 mStrokeIdToPointerId.clear(); 333 } 334 335 private void notifyService(IAccessibilityServiceClient service, int sequence, boolean success) { 336 try { 337 service.onPerformGestureResult(sequence, success); 338 } catch (RemoteException re) { 339 Slog.e(LOG_TAG, "Error sending motion event injection status to " 340 + mServiceInterfaceForCurrentGesture, re); 341 } 342 } 343 344 private List<MotionEvent> getMotionEventsFromGestureSteps( 345 List<GestureStep> steps, long startTime) { 346 final List<MotionEvent> motionEvents = new ArrayList<>(); 347 348 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 349 350 for (int i = 0; i < steps.size(); i++) { 351 GestureDescription.GestureStep step = steps.get(i); 352 int currentTouchPointSize = step.numTouchPoints; 353 if (currentTouchPointSize > lastTouchPoints.length) { 354 mNumLastTouchPoints = 0; 355 motionEvents.clear(); 356 return motionEvents; 357 } 358 359 appendMoveEventIfNeeded(motionEvents, step.touchPoints, currentTouchPointSize, 360 startTime + step.timeSinceGestureStart); 361 appendUpEvents(motionEvents, step.touchPoints, currentTouchPointSize, 362 startTime + step.timeSinceGestureStart); 363 appendDownEvents(motionEvents, step.touchPoints, currentTouchPointSize, 364 startTime + step.timeSinceGestureStart); 365 } 366 return motionEvents; 367 } 368 369 private TouchPoint[] getLastTouchPoints() { 370 if (mLastTouchPoints == null) { 371 int capacity = GestureDescription.getMaxStrokeCount(); 372 mLastTouchPoints = new TouchPoint[capacity]; 373 for (int i = 0; i < capacity; i++) { 374 mLastTouchPoints[i] = new GestureDescription.TouchPoint(); 375 } 376 } 377 return mLastTouchPoints; 378 } 379 380 private void appendMoveEventIfNeeded(List<MotionEvent> motionEvents, 381 TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { 382 /* Look for pointers that have moved */ 383 boolean moveFound = false; 384 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 385 for (int i = 0; i < currentTouchPointsSize; i++) { 386 int lastPointsIndex = findPointByStrokeId(lastTouchPoints, mNumLastTouchPoints, 387 currentTouchPoints[i].mStrokeId); 388 if (lastPointsIndex >= 0) { 389 moveFound |= (lastTouchPoints[lastPointsIndex].mX != currentTouchPoints[i].mX) 390 || (lastTouchPoints[lastPointsIndex].mY != currentTouchPoints[i].mY); 391 lastTouchPoints[lastPointsIndex].copyFrom(currentTouchPoints[i]); 392 } 393 } 394 395 if (moveFound) { 396 motionEvents.add(obtainMotionEvent(mDownTime, currentTime, MotionEvent.ACTION_MOVE, 397 lastTouchPoints, mNumLastTouchPoints)); 398 } 399 } 400 401 private void appendUpEvents(List<MotionEvent> motionEvents, 402 TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { 403 /* Look for a pointer at the end of its path */ 404 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 405 for (int i = 0; i < currentTouchPointsSize; i++) { 406 if (currentTouchPoints[i].mIsEndOfPath) { 407 int indexOfUpEvent = findPointByStrokeId(lastTouchPoints, mNumLastTouchPoints, 408 currentTouchPoints[i].mStrokeId); 409 if (indexOfUpEvent < 0) { 410 continue; // Should not happen 411 } 412 int action = (mNumLastTouchPoints == 1) ? MotionEvent.ACTION_UP 413 : MotionEvent.ACTION_POINTER_UP; 414 action |= indexOfUpEvent << MotionEvent.ACTION_POINTER_INDEX_SHIFT; 415 motionEvents.add(obtainMotionEvent(mDownTime, currentTime, action, 416 lastTouchPoints, mNumLastTouchPoints)); 417 /* Remove this point from lastTouchPoints */ 418 for (int j = indexOfUpEvent; j < mNumLastTouchPoints - 1; j++) { 419 lastTouchPoints[j].copyFrom(mLastTouchPoints[j + 1]); 420 } 421 mNumLastTouchPoints--; 422 if (mNumLastTouchPoints == 0) { 423 mStrokeIdToPointerId.clear(); 424 } 425 } 426 } 427 } 428 429 private void appendDownEvents(List<MotionEvent> motionEvents, 430 TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { 431 /* Look for a pointer that is just starting */ 432 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 433 for (int i = 0; i < currentTouchPointsSize; i++) { 434 if (currentTouchPoints[i].mIsStartOfPath) { 435 /* Add the point to last coords and use the new array to generate the event */ 436 lastTouchPoints[mNumLastTouchPoints++].copyFrom(currentTouchPoints[i]); 437 int action = (mNumLastTouchPoints == 1) ? MotionEvent.ACTION_DOWN 438 : MotionEvent.ACTION_POINTER_DOWN; 439 if (action == MotionEvent.ACTION_DOWN) { 440 mDownTime = currentTime; 441 } 442 action |= i << MotionEvent.ACTION_POINTER_INDEX_SHIFT; 443 motionEvents.add(obtainMotionEvent(mDownTime, currentTime, action, 444 lastTouchPoints, mNumLastTouchPoints)); 445 } 446 } 447 } 448 449 private MotionEvent obtainMotionEvent(long downTime, long eventTime, int action, 450 TouchPoint[] touchPoints, int touchPointsSize) { 451 if ((sPointerCoords == null) || (sPointerCoords.length < touchPointsSize)) { 452 sPointerCoords = new MotionEvent.PointerCoords[touchPointsSize]; 453 for (int i = 0; i < touchPointsSize; i++) { 454 sPointerCoords[i] = new MotionEvent.PointerCoords(); 455 } 456 } 457 if ((sPointerProps == null) || (sPointerProps.length < touchPointsSize)) { 458 sPointerProps = new MotionEvent.PointerProperties[touchPointsSize]; 459 for (int i = 0; i < touchPointsSize; i++) { 460 sPointerProps[i] = new MotionEvent.PointerProperties(); 461 } 462 } 463 for (int i = 0; i < touchPointsSize; i++) { 464 int pointerId = mStrokeIdToPointerId.get(touchPoints[i].mStrokeId, -1); 465 if (pointerId == -1) { 466 pointerId = getUnusedPointerId(); 467 mStrokeIdToPointerId.put(touchPoints[i].mStrokeId, pointerId); 468 } 469 sPointerProps[i].id = pointerId; 470 sPointerProps[i].toolType = MotionEvent.TOOL_TYPE_UNKNOWN; 471 sPointerCoords[i].clear(); 472 sPointerCoords[i].pressure = 1.0f; 473 sPointerCoords[i].size = 1.0f; 474 sPointerCoords[i].x = touchPoints[i].mX; 475 sPointerCoords[i].y = touchPoints[i].mY; 476 } 477 return MotionEvent.obtain(downTime, eventTime, action, touchPointsSize, 478 sPointerProps, sPointerCoords, EVENT_META_STATE, EVENT_BUTTON_STATE, 479 EVENT_X_PRECISION, EVENT_Y_PRECISION, EVENT_DEVICE_ID, EVENT_EDGE_FLAGS, 480 EVENT_SOURCE, EVENT_FLAGS); 481 } 482 483 private static int findPointByStrokeId(TouchPoint[] touchPoints, int touchPointsSize, 484 int strokeId) { 485 for (int i = 0; i < touchPointsSize; i++) { 486 if (touchPoints[i].mStrokeId == strokeId) { 487 return i; 488 } 489 } 490 return -1; 491 } 492 private int getUnusedPointerId() { 493 int MAX_POINTER_ID = 10; 494 int pointerId = 0; 495 while (mStrokeIdToPointerId.indexOfValue(pointerId) >= 0) { 496 pointerId++; 497 if (pointerId >= MAX_POINTER_ID) { 498 return MAX_POINTER_ID; 499 } 500 } 501 return pointerId; 502 } 503 } 504
