1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.accessibility; 18 19 import android.accessibilityservice.GestureDescription; 20 import android.accessibilityservice.GestureDescription.GestureStep; 21 import android.accessibilityservice.GestureDescription.TouchPoint; 22 import android.accessibilityservice.IAccessibilityServiceClient; 23 import android.os.Handler; 24 import android.os.Looper; 25 import android.os.Message; 26 import android.os.RemoteException; 27 import android.os.SystemClock; 28 import android.util.IntArray; 29 import android.util.Slog; 30 import android.util.SparseArray; 31 import android.util.SparseIntArray; 32 import android.view.InputDevice; 33 import android.view.MotionEvent; 34 35 import com.android.internal.os.SomeArgs; 36 import com.android.server.policy.WindowManagerPolicy; 37 38 import java.util.ArrayList; 39 import java.util.Arrays; 40 import java.util.List; 41 42 /** 43 * Injects MotionEvents to permit {@code AccessibilityService}s to touch the screen on behalf of 44 * users. 45 * <p> 46 * All methods except {@code injectEvents} must be called only from the main thread. 47 */ 48 public class MotionEventInjector extends BaseEventStreamTransformation implements Handler.Callback { 49 private static final String LOG_TAG = "MotionEventInjector"; 50 private static final int MESSAGE_SEND_MOTION_EVENT = 1; 51 private static final int MESSAGE_INJECT_EVENTS = 2; 52 53 /** 54 * Constants used to initialize all MotionEvents 55 */ 56 private static final int EVENT_META_STATE = 0; 57 private static final int EVENT_BUTTON_STATE = 0; 58 private static final int EVENT_DEVICE_ID = 0; 59 private static final int EVENT_EDGE_FLAGS = 0; 60 private static final int EVENT_SOURCE = InputDevice.SOURCE_TOUCHSCREEN; 61 private static final int EVENT_FLAGS = 0; 62 private static final float EVENT_X_PRECISION = 1; 63 private static final float EVENT_Y_PRECISION = 1; 64 65 private static MotionEvent.PointerCoords[] sPointerCoords; 66 private static MotionEvent.PointerProperties[] sPointerProps; 67 68 private final Handler mHandler; 69 private final SparseArray<Boolean> mOpenGesturesInProgress = new SparseArray<>(); 70 71 private IAccessibilityServiceClient mServiceInterfaceForCurrentGesture; 72 private IntArray mSequencesInProgress = new IntArray(5); 73 private boolean mIsDestroyed = false; 74 private TouchPoint[] mLastTouchPoints; 75 private int mNumLastTouchPoints; 76 private long mDownTime; 77 private long mLastScheduledEventTime; 78 private SparseIntArray mStrokeIdToPointerId = new SparseIntArray(5); 79 80 /** 81 * @param looper A looper on the main thread to use for dispatching new events 82 */ 83 public MotionEventInjector(Looper looper) { 84 mHandler = new Handler(looper, this); 85 } 86 87 /** 88 * @param handler A handler to post messages. Exposes internal state for testing only. 89 */ 90 public MotionEventInjector(Handler handler) { 91 mHandler = handler; 92 } 93 94 /** 95 * Schedule a gesture for injection. The gesture is defined by a set of {@code GestureStep}s, 96 * from which {@code MotionEvent}s will be derived. All gestures currently in progress will be 97 * cancelled. 98 * 99 * @param gestureSteps The gesture steps to inject. 100 * @param serviceInterface The interface to call back with a result when the gesture is 101 * either complete or cancelled. 102 */ 103 public void injectEvents(List<GestureStep> gestureSteps, 104 IAccessibilityServiceClient serviceInterface, int sequence) { 105 SomeArgs args = SomeArgs.obtain(); 106 args.arg1 = gestureSteps; 107 args.arg2 = serviceInterface; 108 args.argi1 = sequence; 109 mHandler.sendMessage(mHandler.obtainMessage(MESSAGE_INJECT_EVENTS, args)); 110 } 111 112 @Override 113 public void onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) { 114 cancelAnyPendingInjectedEvents(); 115 sendMotionEventToNext(event, rawEvent, policyFlags); 116 } 117 118 @Override 119 public void clearEvents(int inputSource) { 120 /* 121 * Reset state for motion events passing through so we won't send a cancel event for 122 * them. 123 */ 124 if (!mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { 125 mOpenGesturesInProgress.put(inputSource, false); 126 } 127 } 128 129 @Override 130 public void onDestroy() { 131 cancelAnyPendingInjectedEvents(); 132 mIsDestroyed = true; 133 } 134 135 @Override 136 public boolean handleMessage(Message message) { 137 if (message.what == MESSAGE_INJECT_EVENTS) { 138 SomeArgs args = (SomeArgs) message.obj; 139 injectEventsMainThread((List<GestureStep>) args.arg1, 140 (IAccessibilityServiceClient) args.arg2, args.argi1); 141 args.recycle(); 142 return true; 143 } 144 if (message.what != MESSAGE_SEND_MOTION_EVENT) { 145 Slog.e(LOG_TAG, "Unknown message: " + message.what); 146 return false; 147 } 148 MotionEvent motionEvent = (MotionEvent) message.obj; 149 sendMotionEventToNext(motionEvent, motionEvent, WindowManagerPolicy.FLAG_PASS_TO_USER); 150 boolean isEndOfSequence = message.arg1 != 0; 151 if (isEndOfSequence) { 152 notifyService(mServiceInterfaceForCurrentGesture, mSequencesInProgress.get(0), true); 153 mSequencesInProgress.remove(0); 154 } 155 return true; 156 } 157 158 private void injectEventsMainThread(List<GestureStep> gestureSteps, 159 IAccessibilityServiceClient serviceInterface, int sequence) { 160 if (mIsDestroyed) { 161 try { 162 serviceInterface.onPerformGestureResult(sequence, false); 163 } catch (RemoteException re) { 164 Slog.e(LOG_TAG, "Error sending status with mIsDestroyed to " + serviceInterface, 165 re); 166 } 167 return; 168 } 169 170 if (getNext() == null) { 171 notifyService(serviceInterface, sequence, false); 172 return; 173 } 174 175 boolean continuingGesture = newGestureTriesToContinueOldOne(gestureSteps); 176 177 if (continuingGesture) { 178 if ((serviceInterface != mServiceInterfaceForCurrentGesture) 179 || !prepareToContinueOldGesture(gestureSteps)) { 180 cancelAnyPendingInjectedEvents(); 181 notifyService(serviceInterface, sequence, false); 182 return; 183 } 184 } 185 if (!continuingGesture) { 186 cancelAnyPendingInjectedEvents(); 187 // Injected gestures have been canceled, but real gestures still need cancelling 188 cancelAnyGestureInProgress(EVENT_SOURCE); 189 } 190 mServiceInterfaceForCurrentGesture = serviceInterface; 191 192 long currentTime = SystemClock.uptimeMillis(); 193 List<MotionEvent> events = getMotionEventsFromGestureSteps(gestureSteps, 194 (mSequencesInProgress.size() == 0) ? currentTime : mLastScheduledEventTime); 195 if (events.isEmpty()) { 196 notifyService(serviceInterface, sequence, false); 197 return; 198 } 199 mSequencesInProgress.add(sequence); 200 201 for (int i = 0; i < events.size(); i++) { 202 MotionEvent event = events.get(i); 203 int isEndOfSequence = (i == events.size() - 1) ? 1 : 0; 204 Message message = mHandler.obtainMessage( 205 MESSAGE_SEND_MOTION_EVENT, isEndOfSequence, 0, event); 206 mLastScheduledEventTime = event.getEventTime(); 207 mHandler.sendMessageDelayed(message, Math.max(0, event.getEventTime() - currentTime)); 208 } 209 } 210 211 private boolean newGestureTriesToContinueOldOne(List<GestureStep> gestureSteps) { 212 if (gestureSteps.isEmpty()) { 213 return false; 214 } 215 GestureStep firstStep = gestureSteps.get(0); 216 for (int i = 0; i < firstStep.numTouchPoints; i++) { 217 if (!firstStep.touchPoints[i].mIsStartOfPath) { 218 return true; 219 } 220 } 221 return false; 222 } 223 224 /** 225 * A gesture can only continue a gesture if it contains intermediate points that continue 226 * each continued stroke of the last gesture, and no extra points. 227 * 228 * @param gestureSteps The steps of the new gesture 229 * @return {@code true} if the new gesture could continue the last one dispatched. {@code false} 230 * otherwise. 231 */ 232 private boolean prepareToContinueOldGesture(List<GestureStep> gestureSteps) { 233 if (gestureSteps.isEmpty() || (mLastTouchPoints == null) || (mNumLastTouchPoints == 0)) { 234 return false; 235 } 236 GestureStep firstStep = gestureSteps.get(0); 237 // Make sure all of the continuing paths match up 238 int numContinuedStrokes = 0; 239 for (int i = 0; i < firstStep.numTouchPoints; i++) { 240 TouchPoint touchPoint = firstStep.touchPoints[i]; 241 if (!touchPoint.mIsStartOfPath) { 242 int continuedPointerId = mStrokeIdToPointerId 243 .get(touchPoint.mContinuedStrokeId, -1); 244 if (continuedPointerId == -1) { 245 Slog.w(LOG_TAG, "Can't continue gesture due to unknown continued stroke id in " 246 + touchPoint); 247 return false; 248 } 249 mStrokeIdToPointerId.put(touchPoint.mStrokeId, continuedPointerId); 250 int lastPointIndex = findPointByStrokeId( 251 mLastTouchPoints, mNumLastTouchPoints, touchPoint.mContinuedStrokeId); 252 if (lastPointIndex < 0) { 253 Slog.w(LOG_TAG, "Can't continue gesture due continued gesture id of " 254 + touchPoint + " not matching any previous strokes in " 255 + Arrays.asList(mLastTouchPoints)); 256 return false; 257 } 258 if (mLastTouchPoints[lastPointIndex].mIsEndOfPath 259 || (mLastTouchPoints[lastPointIndex].mX != touchPoint.mX) 260 || (mLastTouchPoints[lastPointIndex].mY != touchPoint.mY)) { 261 Slog.w(LOG_TAG, "Can't continue gesture due to points mismatch between " 262 + mLastTouchPoints[lastPointIndex] + " and " + touchPoint); 263 return false; 264 } 265 // Update the last touch point to match the continuation, so the gestures will 266 // line up 267 mLastTouchPoints[lastPointIndex].mStrokeId = touchPoint.mStrokeId; 268 } 269 numContinuedStrokes++; 270 } 271 // Make sure we didn't miss any paths 272 for (int i = 0; i < mNumLastTouchPoints; i++) { 273 if (!mLastTouchPoints[i].mIsEndOfPath) { 274 numContinuedStrokes--; 275 } 276 } 277 return numContinuedStrokes == 0; 278 } 279 280 private void sendMotionEventToNext(MotionEvent event, MotionEvent rawEvent, 281 int policyFlags) { 282 if (getNext() != null) { 283 super.onMotionEvent(event, rawEvent, policyFlags); 284 if (event.getActionMasked() == MotionEvent.ACTION_DOWN) { 285 mOpenGesturesInProgress.put(event.getSource(), true); 286 } 287 if ((event.getActionMasked() == MotionEvent.ACTION_UP) 288 || (event.getActionMasked() == MotionEvent.ACTION_CANCEL)) { 289 mOpenGesturesInProgress.put(event.getSource(), false); 290 } 291 } 292 } 293 294 private void cancelAnyGestureInProgress(int source) { 295 if ((getNext() != null) && mOpenGesturesInProgress.get(source, false)) { 296 long now = SystemClock.uptimeMillis(); 297 MotionEvent cancelEvent = 298 obtainMotionEvent(now, now, MotionEvent.ACTION_CANCEL, getLastTouchPoints(), 1); 299 sendMotionEventToNext(cancelEvent, cancelEvent, 300 WindowManagerPolicy.FLAG_PASS_TO_USER); 301 mOpenGesturesInProgress.put(source, false); 302 } 303 } 304 305 private void cancelAnyPendingInjectedEvents() { 306 if (mHandler.hasMessages(MESSAGE_SEND_MOTION_EVENT)) { 307 mHandler.removeMessages(MESSAGE_SEND_MOTION_EVENT); 308 cancelAnyGestureInProgress(EVENT_SOURCE); 309 for (int i = mSequencesInProgress.size() - 1; i >= 0; i--) { 310 notifyService(mServiceInterfaceForCurrentGesture, 311 mSequencesInProgress.get(i), false); 312 mSequencesInProgress.remove(i); 313 } 314 } else if (mNumLastTouchPoints != 0) { 315 // An injected gesture is in progress and waiting for a continuation. Cancel it. 316 cancelAnyGestureInProgress(EVENT_SOURCE); 317 } 318 mNumLastTouchPoints = 0; 319 mStrokeIdToPointerId.clear(); 320 } 321 322 private void notifyService(IAccessibilityServiceClient service, int sequence, boolean success) { 323 try { 324 service.onPerformGestureResult(sequence, success); 325 } catch (RemoteException re) { 326 Slog.e(LOG_TAG, "Error sending motion event injection status to " 327 + mServiceInterfaceForCurrentGesture, re); 328 } 329 } 330 331 private List<MotionEvent> getMotionEventsFromGestureSteps( 332 List<GestureStep> steps, long startTime) { 333 final List<MotionEvent> motionEvents = new ArrayList<>(); 334 335 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 336 337 for (int i = 0; i < steps.size(); i++) { 338 GestureDescription.GestureStep step = steps.get(i); 339 int currentTouchPointSize = step.numTouchPoints; 340 if (currentTouchPointSize > lastTouchPoints.length) { 341 mNumLastTouchPoints = 0; 342 motionEvents.clear(); 343 return motionEvents; 344 } 345 346 appendMoveEventIfNeeded(motionEvents, step.touchPoints, currentTouchPointSize, 347 startTime + step.timeSinceGestureStart); 348 appendUpEvents(motionEvents, step.touchPoints, currentTouchPointSize, 349 startTime + step.timeSinceGestureStart); 350 appendDownEvents(motionEvents, step.touchPoints, currentTouchPointSize, 351 startTime + step.timeSinceGestureStart); 352 } 353 return motionEvents; 354 } 355 356 private TouchPoint[] getLastTouchPoints() { 357 if (mLastTouchPoints == null) { 358 int capacity = GestureDescription.getMaxStrokeCount(); 359 mLastTouchPoints = new TouchPoint[capacity]; 360 for (int i = 0; i < capacity; i++) { 361 mLastTouchPoints[i] = new GestureDescription.TouchPoint(); 362 } 363 } 364 return mLastTouchPoints; 365 } 366 367 private void appendMoveEventIfNeeded(List<MotionEvent> motionEvents, 368 TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { 369 /* Look for pointers that have moved */ 370 boolean moveFound = false; 371 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 372 for (int i = 0; i < currentTouchPointsSize; i++) { 373 int lastPointsIndex = findPointByStrokeId(lastTouchPoints, mNumLastTouchPoints, 374 currentTouchPoints[i].mStrokeId); 375 if (lastPointsIndex >= 0) { 376 moveFound |= (lastTouchPoints[lastPointsIndex].mX != currentTouchPoints[i].mX) 377 || (lastTouchPoints[lastPointsIndex].mY != currentTouchPoints[i].mY); 378 lastTouchPoints[lastPointsIndex].copyFrom(currentTouchPoints[i]); 379 } 380 } 381 382 if (moveFound) { 383 motionEvents.add(obtainMotionEvent(mDownTime, currentTime, MotionEvent.ACTION_MOVE, 384 lastTouchPoints, mNumLastTouchPoints)); 385 } 386 } 387 388 private void appendUpEvents(List<MotionEvent> motionEvents, 389 TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { 390 /* Look for a pointer at the end of its path */ 391 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 392 for (int i = 0; i < currentTouchPointsSize; i++) { 393 if (currentTouchPoints[i].mIsEndOfPath) { 394 int indexOfUpEvent = findPointByStrokeId(lastTouchPoints, mNumLastTouchPoints, 395 currentTouchPoints[i].mStrokeId); 396 if (indexOfUpEvent < 0) { 397 continue; // Should not happen 398 } 399 int action = (mNumLastTouchPoints == 1) ? MotionEvent.ACTION_UP 400 : MotionEvent.ACTION_POINTER_UP; 401 action |= indexOfUpEvent << MotionEvent.ACTION_POINTER_INDEX_SHIFT; 402 motionEvents.add(obtainMotionEvent(mDownTime, currentTime, action, 403 lastTouchPoints, mNumLastTouchPoints)); 404 /* Remove this point from lastTouchPoints */ 405 for (int j = indexOfUpEvent; j < mNumLastTouchPoints - 1; j++) { 406 lastTouchPoints[j].copyFrom(mLastTouchPoints[j + 1]); 407 } 408 mNumLastTouchPoints--; 409 if (mNumLastTouchPoints == 0) { 410 mStrokeIdToPointerId.clear(); 411 } 412 } 413 } 414 } 415 416 private void appendDownEvents(List<MotionEvent> motionEvents, 417 TouchPoint[] currentTouchPoints, int currentTouchPointsSize, long currentTime) { 418 /* Look for a pointer that is just starting */ 419 TouchPoint[] lastTouchPoints = getLastTouchPoints(); 420 for (int i = 0; i < currentTouchPointsSize; i++) { 421 if (currentTouchPoints[i].mIsStartOfPath) { 422 /* Add the point to last coords and use the new array to generate the event */ 423 lastTouchPoints[mNumLastTouchPoints++].copyFrom(currentTouchPoints[i]); 424 int action = (mNumLastTouchPoints == 1) ? MotionEvent.ACTION_DOWN 425 : MotionEvent.ACTION_POINTER_DOWN; 426 if (action == MotionEvent.ACTION_DOWN) { 427 mDownTime = currentTime; 428 } 429 action |= i << MotionEvent.ACTION_POINTER_INDEX_SHIFT; 430 motionEvents.add(obtainMotionEvent(mDownTime, currentTime, action, 431 lastTouchPoints, mNumLastTouchPoints)); 432 } 433 } 434 } 435 436 private MotionEvent obtainMotionEvent(long downTime, long eventTime, int action, 437 TouchPoint[] touchPoints, int touchPointsSize) { 438 if ((sPointerCoords == null) || (sPointerCoords.length < touchPointsSize)) { 439 sPointerCoords = new MotionEvent.PointerCoords[touchPointsSize]; 440 for (int i = 0; i < touchPointsSize; i++) { 441 sPointerCoords[i] = new MotionEvent.PointerCoords(); 442 } 443 } 444 if ((sPointerProps == null) || (sPointerProps.length < touchPointsSize)) { 445 sPointerProps = new MotionEvent.PointerProperties[touchPointsSize]; 446 for (int i = 0; i < touchPointsSize; i++) { 447 sPointerProps[i] = new MotionEvent.PointerProperties(); 448 } 449 } 450 for (int i = 0; i < touchPointsSize; i++) { 451 int pointerId = mStrokeIdToPointerId.get(touchPoints[i].mStrokeId, -1); 452 if (pointerId == -1) { 453 pointerId = getUnusedPointerId(); 454 mStrokeIdToPointerId.put(touchPoints[i].mStrokeId, pointerId); 455 } 456 sPointerProps[i].id = pointerId; 457 sPointerProps[i].toolType = MotionEvent.TOOL_TYPE_UNKNOWN; 458 sPointerCoords[i].clear(); 459 sPointerCoords[i].pressure = 1.0f; 460 sPointerCoords[i].size = 1.0f; 461 sPointerCoords[i].x = touchPoints[i].mX; 462 sPointerCoords[i].y = touchPoints[i].mY; 463 } 464 return MotionEvent.obtain(downTime, eventTime, action, touchPointsSize, 465 sPointerProps, sPointerCoords, EVENT_META_STATE, EVENT_BUTTON_STATE, 466 EVENT_X_PRECISION, EVENT_Y_PRECISION, EVENT_DEVICE_ID, EVENT_EDGE_FLAGS, 467 EVENT_SOURCE, EVENT_FLAGS); 468 } 469 470 private static int findPointByStrokeId(TouchPoint[] touchPoints, int touchPointsSize, 471 int strokeId) { 472 for (int i = 0; i < touchPointsSize; i++) { 473 if (touchPoints[i].mStrokeId == strokeId) { 474 return i; 475 } 476 } 477 return -1; 478 } 479 private int getUnusedPointerId() { 480 int MAX_POINTER_ID = 10; 481 int pointerId = 0; 482 while (mStrokeIdToPointerId.indexOfValue(pointerId) >= 0) { 483 pointerId++; 484 if (pointerId >= MAX_POINTER_ID) { 485 return MAX_POINTER_ID; 486 } 487 } 488 return pointerId; 489 } 490 } 491
