1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.soundtrigger; 18 19 import static android.Manifest.permission.BIND_SOUND_TRIGGER_DETECTION_SERVICE; 20 import static android.content.Context.BIND_AUTO_CREATE; 21 import static android.content.Context.BIND_FOREGROUND_SERVICE; 22 import static android.content.pm.PackageManager.GET_META_DATA; 23 import static android.content.pm.PackageManager.GET_SERVICES; 24 import static android.content.pm.PackageManager.MATCH_DEBUG_TRIAGED_MISSING; 25 import static android.hardware.soundtrigger.SoundTrigger.STATUS_ERROR; 26 import static android.hardware.soundtrigger.SoundTrigger.STATUS_OK; 27 import static android.provider.Settings.Global.MAX_SOUND_TRIGGER_DETECTION_SERVICE_OPS_PER_DAY; 28 import static android.provider.Settings.Global.SOUND_TRIGGER_DETECTION_SERVICE_OP_TIMEOUT; 29 30 import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage; 31 32 import android.Manifest; 33 import android.annotation.NonNull; 34 import android.annotation.Nullable; 35 import android.app.PendingIntent; 36 import android.content.ComponentName; 37 import android.content.Context; 38 import android.content.Intent; 39 import android.content.ServiceConnection; 40 import android.content.pm.PackageManager; 41 import android.content.pm.ResolveInfo; 42 import android.hardware.soundtrigger.IRecognitionStatusCallback; 43 import android.hardware.soundtrigger.SoundTrigger; 44 import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; 45 import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; 46 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; 47 import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; 48 import android.hardware.soundtrigger.SoundTrigger.SoundModel; 49 import android.media.AudioAttributes; 50 import android.media.AudioFormat; 51 import android.media.AudioRecord; 52 import android.media.MediaRecorder; 53 import android.media.soundtrigger.ISoundTriggerDetectionService; 54 import android.media.soundtrigger.ISoundTriggerDetectionServiceClient; 55 import android.media.soundtrigger.SoundTriggerDetectionService; 56 import android.media.soundtrigger.SoundTriggerManager; 57 import android.os.Binder; 58 import android.os.Bundle; 59 import android.os.Handler; 60 import android.os.IBinder; 61 import android.os.Looper; 62 import android.os.Parcel; 63 import android.os.ParcelUuid; 64 import android.os.PowerManager; 65 import android.os.RemoteException; 66 import android.os.UserHandle; 67 import android.provider.Settings; 68 import android.util.ArrayMap; 69 import android.util.ArraySet; 70 import android.util.Slog; 71 72 import com.android.internal.annotations.GuardedBy; 73 import com.android.internal.app.ISoundTriggerService; 74 import com.android.internal.util.DumpUtils; 75 import com.android.internal.util.Preconditions; 76 import com.android.server.SystemService; 77 78 import java.io.FileDescriptor; 79 import java.io.PrintWriter; 80 import java.util.ArrayList; 81 import java.util.TreeMap; 82 import java.util.UUID; 83 import java.util.concurrent.TimeUnit; 84 85 /** 86 * A single SystemService to manage all sound/voice-based sound models on the DSP. 87 * This services provides apis to manage sound trigger-based sound models via 88 * the ISoundTriggerService interface. This class also publishes a local interface encapsulating 89 * the functionality provided by {@link SoundTriggerHelper} for use by 90 * {@link VoiceInteractionManagerService}. 91 * 92 * @hide 93 */ 94 public class SoundTriggerService extends SystemService { 95 private static final String TAG = "SoundTriggerService"; 96 private static final boolean DEBUG = true; 97 98 final Context mContext; 99 private Object mLock; 100 private final SoundTriggerServiceStub mServiceStub; 101 private final LocalSoundTriggerService mLocalSoundTriggerService; 102 private SoundTriggerDbHelper mDbHelper; 103 private SoundTriggerHelper mSoundTriggerHelper; 104 private final TreeMap<UUID, SoundModel> mLoadedModels; 105 private Object mCallbacksLock; 106 private final TreeMap<UUID, IRecognitionStatusCallback> mCallbacks; 107 private PowerManager.WakeLock mWakelock; 108 109 /** Number of ops run by the {@link RemoteSoundTriggerDetectionService} per package name */ 110 @GuardedBy("mLock") 111 private final ArrayMap<String, NumOps> mNumOpsPerPackage = new ArrayMap<>(); 112 113 public SoundTriggerService(Context context) { 114 super(context); 115 mContext = context; 116 mServiceStub = new SoundTriggerServiceStub(); 117 mLocalSoundTriggerService = new LocalSoundTriggerService(context); 118 mLoadedModels = new TreeMap<UUID, SoundModel>(); 119 mCallbacksLock = new Object(); 120 mCallbacks = new TreeMap<>(); 121 mLock = new Object(); 122 } 123 124 @Override 125 public void onStart() { 126 publishBinderService(Context.SOUND_TRIGGER_SERVICE, mServiceStub); 127 publishLocalService(SoundTriggerInternal.class, mLocalSoundTriggerService); 128 } 129 130 @Override 131 public void onBootPhase(int phase) { 132 if (PHASE_SYSTEM_SERVICES_READY == phase) { 133 initSoundTriggerHelper(); 134 mLocalSoundTriggerService.setSoundTriggerHelper(mSoundTriggerHelper); 135 } else if (PHASE_THIRD_PARTY_APPS_CAN_START == phase) { 136 mDbHelper = new SoundTriggerDbHelper(mContext); 137 } 138 } 139 140 @Override 141 public void onStartUser(int userHandle) { 142 } 143 144 @Override 145 public void onSwitchUser(int userHandle) { 146 } 147 148 private synchronized void initSoundTriggerHelper() { 149 if (mSoundTriggerHelper == null) { 150 mSoundTriggerHelper = new SoundTriggerHelper(mContext); 151 } 152 } 153 154 private synchronized boolean isInitialized() { 155 if (mSoundTriggerHelper == null ) { 156 Slog.e(TAG, "SoundTriggerHelper not initialized."); 157 return false; 158 } 159 return true; 160 } 161 162 class SoundTriggerServiceStub extends ISoundTriggerService.Stub { 163 @Override 164 public boolean onTransact(int code, Parcel data, Parcel reply, int flags) 165 throws RemoteException { 166 try { 167 return super.onTransact(code, data, reply, flags); 168 } catch (RuntimeException e) { 169 // The activity manager only throws security exceptions, so let's 170 // log all others. 171 if (!(e instanceof SecurityException)) { 172 Slog.wtf(TAG, "SoundTriggerService Crash", e); 173 } 174 throw e; 175 } 176 } 177 178 @Override 179 public int startRecognition(ParcelUuid parcelUuid, IRecognitionStatusCallback callback, 180 RecognitionConfig config) { 181 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 182 if (!isInitialized()) return STATUS_ERROR; 183 if (DEBUG) { 184 Slog.i(TAG, "startRecognition(): Uuid : " + parcelUuid); 185 } 186 187 GenericSoundModel model = getSoundModel(parcelUuid); 188 if (model == null) { 189 Slog.e(TAG, "Null model in database for id: " + parcelUuid); 190 return STATUS_ERROR; 191 } 192 193 return mSoundTriggerHelper.startGenericRecognition(parcelUuid.getUuid(), model, 194 callback, config); 195 } 196 197 @Override 198 public int stopRecognition(ParcelUuid parcelUuid, IRecognitionStatusCallback callback) { 199 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 200 if (DEBUG) { 201 Slog.i(TAG, "stopRecognition(): Uuid : " + parcelUuid); 202 } 203 if (!isInitialized()) return STATUS_ERROR; 204 return mSoundTriggerHelper.stopGenericRecognition(parcelUuid.getUuid(), callback); 205 } 206 207 @Override 208 public SoundTrigger.GenericSoundModel getSoundModel(ParcelUuid soundModelId) { 209 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 210 if (DEBUG) { 211 Slog.i(TAG, "getSoundModel(): id = " + soundModelId); 212 } 213 SoundTrigger.GenericSoundModel model = mDbHelper.getGenericSoundModel( 214 soundModelId.getUuid()); 215 return model; 216 } 217 218 @Override 219 public void updateSoundModel(SoundTrigger.GenericSoundModel soundModel) { 220 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 221 if (DEBUG) { 222 Slog.i(TAG, "updateSoundModel(): model = " + soundModel); 223 } 224 mDbHelper.updateGenericSoundModel(soundModel); 225 } 226 227 @Override 228 public void deleteSoundModel(ParcelUuid soundModelId) { 229 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 230 if (DEBUG) { 231 Slog.i(TAG, "deleteSoundModel(): id = " + soundModelId); 232 } 233 // Unload the model if it is loaded. 234 mSoundTriggerHelper.unloadGenericSoundModel(soundModelId.getUuid()); 235 mDbHelper.deleteGenericSoundModel(soundModelId.getUuid()); 236 } 237 238 @Override 239 public int loadGenericSoundModel(GenericSoundModel soundModel) { 240 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 241 if (!isInitialized()) return STATUS_ERROR; 242 if (soundModel == null || soundModel.uuid == null) { 243 Slog.e(TAG, "Invalid sound model"); 244 return STATUS_ERROR; 245 } 246 if (DEBUG) { 247 Slog.i(TAG, "loadGenericSoundModel(): id = " + soundModel.uuid); 248 } 249 synchronized (mLock) { 250 SoundModel oldModel = mLoadedModels.get(soundModel.uuid); 251 // If the model we're loading is actually different than what we had loaded, we 252 // should unload that other model now. We don't care about return codes since we 253 // don't know if the other model is loaded. 254 if (oldModel != null && !oldModel.equals(soundModel)) { 255 mSoundTriggerHelper.unloadGenericSoundModel(soundModel.uuid); 256 synchronized (mCallbacksLock) { 257 mCallbacks.remove(soundModel.uuid); 258 } 259 } 260 mLoadedModels.put(soundModel.uuid, soundModel); 261 } 262 return STATUS_OK; 263 } 264 265 @Override 266 public int loadKeyphraseSoundModel(KeyphraseSoundModel soundModel) { 267 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 268 if (!isInitialized()) return STATUS_ERROR; 269 if (soundModel == null || soundModel.uuid == null) { 270 Slog.e(TAG, "Invalid sound model"); 271 return STATUS_ERROR; 272 } 273 if (soundModel.keyphrases == null || soundModel.keyphrases.length != 1) { 274 Slog.e(TAG, "Only one keyphrase per model is currently supported."); 275 return STATUS_ERROR; 276 } 277 if (DEBUG) { 278 Slog.i(TAG, "loadKeyphraseSoundModel(): id = " + soundModel.uuid); 279 } 280 synchronized (mLock) { 281 SoundModel oldModel = mLoadedModels.get(soundModel.uuid); 282 // If the model we're loading is actually different than what we had loaded, we 283 // should unload that other model now. We don't care about return codes since we 284 // don't know if the other model is loaded. 285 if (oldModel != null && !oldModel.equals(soundModel)) { 286 mSoundTriggerHelper.unloadKeyphraseSoundModel(soundModel.keyphrases[0].id); 287 synchronized (mCallbacksLock) { 288 mCallbacks.remove(soundModel.uuid); 289 } 290 } 291 mLoadedModels.put(soundModel.uuid, soundModel); 292 } 293 return STATUS_OK; 294 } 295 296 @Override 297 public int startRecognitionForService(ParcelUuid soundModelId, Bundle params, 298 ComponentName detectionService, SoundTrigger.RecognitionConfig config) { 299 Preconditions.checkNotNull(soundModelId); 300 Preconditions.checkNotNull(detectionService); 301 Preconditions.checkNotNull(config); 302 303 return startRecognitionForInt(soundModelId, 304 new RemoteSoundTriggerDetectionService(soundModelId.getUuid(), 305 params, detectionService, Binder.getCallingUserHandle(), config), config); 306 307 } 308 309 @Override 310 public int startRecognitionForIntent(ParcelUuid soundModelId, PendingIntent callbackIntent, 311 SoundTrigger.RecognitionConfig config) { 312 return startRecognitionForInt(soundModelId, 313 new LocalSoundTriggerRecognitionStatusIntentCallback(soundModelId.getUuid(), 314 callbackIntent, config), config); 315 } 316 317 private int startRecognitionForInt(ParcelUuid soundModelId, 318 IRecognitionStatusCallback callback, SoundTrigger.RecognitionConfig config) { 319 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 320 if (!isInitialized()) return STATUS_ERROR; 321 if (DEBUG) { 322 Slog.i(TAG, "startRecognition(): id = " + soundModelId); 323 } 324 325 synchronized (mLock) { 326 SoundModel soundModel = mLoadedModels.get(soundModelId.getUuid()); 327 if (soundModel == null) { 328 Slog.e(TAG, soundModelId + " is not loaded"); 329 return STATUS_ERROR; 330 } 331 IRecognitionStatusCallback existingCallback = null; 332 synchronized (mCallbacksLock) { 333 existingCallback = mCallbacks.get(soundModelId.getUuid()); 334 } 335 if (existingCallback != null) { 336 Slog.e(TAG, soundModelId + " is already running"); 337 return STATUS_ERROR; 338 } 339 int ret; 340 switch (soundModel.type) { 341 case SoundModel.TYPE_KEYPHRASE: { 342 KeyphraseSoundModel keyphraseSoundModel = (KeyphraseSoundModel) soundModel; 343 ret = mSoundTriggerHelper.startKeyphraseRecognition( 344 keyphraseSoundModel.keyphrases[0].id, keyphraseSoundModel, callback, 345 config); 346 } break; 347 case SoundModel.TYPE_GENERIC_SOUND: 348 ret = mSoundTriggerHelper.startGenericRecognition(soundModel.uuid, 349 (GenericSoundModel) soundModel, callback, config); 350 break; 351 default: 352 Slog.e(TAG, "Unknown model type"); 353 return STATUS_ERROR; 354 } 355 356 if (ret != STATUS_OK) { 357 Slog.e(TAG, "Failed to start model: " + ret); 358 return ret; 359 } 360 synchronized (mCallbacksLock) { 361 mCallbacks.put(soundModelId.getUuid(), callback); 362 } 363 } 364 return STATUS_OK; 365 } 366 367 @Override 368 public int stopRecognitionForIntent(ParcelUuid soundModelId) { 369 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 370 if (!isInitialized()) return STATUS_ERROR; 371 if (DEBUG) { 372 Slog.i(TAG, "stopRecognition(): id = " + soundModelId); 373 } 374 375 synchronized (mLock) { 376 SoundModel soundModel = mLoadedModels.get(soundModelId.getUuid()); 377 if (soundModel == null) { 378 Slog.e(TAG, soundModelId + " is not loaded"); 379 return STATUS_ERROR; 380 } 381 IRecognitionStatusCallback callback = null; 382 synchronized (mCallbacksLock) { 383 callback = mCallbacks.get(soundModelId.getUuid()); 384 } 385 if (callback == null) { 386 Slog.e(TAG, soundModelId + " is not running"); 387 return STATUS_ERROR; 388 } 389 int ret; 390 switch (soundModel.type) { 391 case SoundModel.TYPE_KEYPHRASE: 392 ret = mSoundTriggerHelper.stopKeyphraseRecognition( 393 ((KeyphraseSoundModel)soundModel).keyphrases[0].id, callback); 394 break; 395 case SoundModel.TYPE_GENERIC_SOUND: 396 ret = mSoundTriggerHelper.stopGenericRecognition(soundModel.uuid, callback); 397 break; 398 default: 399 Slog.e(TAG, "Unknown model type"); 400 return STATUS_ERROR; 401 } 402 403 if (ret != STATUS_OK) { 404 Slog.e(TAG, "Failed to stop model: " + ret); 405 return ret; 406 } 407 synchronized (mCallbacksLock) { 408 mCallbacks.remove(soundModelId.getUuid()); 409 } 410 } 411 return STATUS_OK; 412 } 413 414 @Override 415 public int unloadSoundModel(ParcelUuid soundModelId) { 416 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 417 if (!isInitialized()) return STATUS_ERROR; 418 if (DEBUG) { 419 Slog.i(TAG, "unloadSoundModel(): id = " + soundModelId); 420 } 421 422 synchronized (mLock) { 423 SoundModel soundModel = mLoadedModels.get(soundModelId.getUuid()); 424 if (soundModel == null) { 425 Slog.e(TAG, soundModelId + " is not loaded"); 426 return STATUS_ERROR; 427 } 428 int ret; 429 switch (soundModel.type) { 430 case SoundModel.TYPE_KEYPHRASE: 431 ret = mSoundTriggerHelper.unloadKeyphraseSoundModel( 432 ((KeyphraseSoundModel)soundModel).keyphrases[0].id); 433 break; 434 case SoundModel.TYPE_GENERIC_SOUND: 435 ret = mSoundTriggerHelper.unloadGenericSoundModel(soundModel.uuid); 436 break; 437 default: 438 Slog.e(TAG, "Unknown model type"); 439 return STATUS_ERROR; 440 } 441 if (ret != STATUS_OK) { 442 Slog.e(TAG, "Failed to unload model"); 443 return ret; 444 } 445 mLoadedModels.remove(soundModelId.getUuid()); 446 return STATUS_OK; 447 } 448 } 449 450 @Override 451 public boolean isRecognitionActive(ParcelUuid parcelUuid) { 452 enforceCallingPermission(Manifest.permission.MANAGE_SOUND_TRIGGER); 453 if (!isInitialized()) return false; 454 synchronized (mCallbacksLock) { 455 IRecognitionStatusCallback callback = mCallbacks.get(parcelUuid.getUuid()); 456 if (callback == null) { 457 return false; 458 } 459 } 460 return mSoundTriggerHelper.isRecognitionRequested(parcelUuid.getUuid()); 461 } 462 } 463 464 private final class LocalSoundTriggerRecognitionStatusIntentCallback 465 extends IRecognitionStatusCallback.Stub { 466 private UUID mUuid; 467 private PendingIntent mCallbackIntent; 468 private RecognitionConfig mRecognitionConfig; 469 470 public LocalSoundTriggerRecognitionStatusIntentCallback(UUID modelUuid, 471 PendingIntent callbackIntent, 472 RecognitionConfig config) { 473 mUuid = modelUuid; 474 mCallbackIntent = callbackIntent; 475 mRecognitionConfig = config; 476 } 477 478 @Override 479 public boolean pingBinder() { 480 return mCallbackIntent != null; 481 } 482 483 @Override 484 public void onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent event) { 485 if (mCallbackIntent == null) { 486 return; 487 } 488 grabWakeLock(); 489 490 Slog.w(TAG, "Keyphrase sound trigger event: " + event); 491 Intent extras = new Intent(); 492 extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, 493 SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_EVENT); 494 extras.putExtra(SoundTriggerManager.EXTRA_RECOGNITION_EVENT, event); 495 try { 496 mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); 497 if (!mRecognitionConfig.allowMultipleTriggers) { 498 removeCallback(/*releaseWakeLock=*/false); 499 } 500 } catch (PendingIntent.CanceledException e) { 501 removeCallback(/*releaseWakeLock=*/true); 502 } 503 } 504 505 @Override 506 public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { 507 if (mCallbackIntent == null) { 508 return; 509 } 510 grabWakeLock(); 511 512 Slog.w(TAG, "Generic sound trigger event: " + event); 513 Intent extras = new Intent(); 514 extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, 515 SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_EVENT); 516 extras.putExtra(SoundTriggerManager.EXTRA_RECOGNITION_EVENT, event); 517 try { 518 mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); 519 if (!mRecognitionConfig.allowMultipleTriggers) { 520 removeCallback(/*releaseWakeLock=*/false); 521 } 522 } catch (PendingIntent.CanceledException e) { 523 removeCallback(/*releaseWakeLock=*/true); 524 } 525 } 526 527 @Override 528 public void onError(int status) { 529 if (mCallbackIntent == null) { 530 return; 531 } 532 grabWakeLock(); 533 534 Slog.i(TAG, "onError: " + status); 535 Intent extras = new Intent(); 536 extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, 537 SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_ERROR); 538 extras.putExtra(SoundTriggerManager.EXTRA_STATUS, status); 539 try { 540 mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); 541 // Remove the callback, but wait for the intent to finish before we let go of the 542 // wake lock 543 removeCallback(/*releaseWakeLock=*/false); 544 } catch (PendingIntent.CanceledException e) { 545 removeCallback(/*releaseWakeLock=*/true); 546 } 547 } 548 549 @Override 550 public void onRecognitionPaused() { 551 if (mCallbackIntent == null) { 552 return; 553 } 554 grabWakeLock(); 555 556 Slog.i(TAG, "onRecognitionPaused"); 557 Intent extras = new Intent(); 558 extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, 559 SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_PAUSED); 560 try { 561 mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); 562 } catch (PendingIntent.CanceledException e) { 563 removeCallback(/*releaseWakeLock=*/true); 564 } 565 } 566 567 @Override 568 public void onRecognitionResumed() { 569 if (mCallbackIntent == null) { 570 return; 571 } 572 grabWakeLock(); 573 574 Slog.i(TAG, "onRecognitionResumed"); 575 Intent extras = new Intent(); 576 extras.putExtra(SoundTriggerManager.EXTRA_MESSAGE_TYPE, 577 SoundTriggerManager.FLAG_MESSAGE_TYPE_RECOGNITION_RESUMED); 578 try { 579 mCallbackIntent.send(mContext, 0, extras, mCallbackCompletedHandler, null); 580 } catch (PendingIntent.CanceledException e) { 581 removeCallback(/*releaseWakeLock=*/true); 582 } 583 } 584 585 private void removeCallback(boolean releaseWakeLock) { 586 mCallbackIntent = null; 587 synchronized (mCallbacksLock) { 588 mCallbacks.remove(mUuid); 589 if (releaseWakeLock) { 590 mWakelock.release(); 591 } 592 } 593 } 594 } 595 596 /** 597 * Counts the number of operations added in the last 24 hours. 598 */ 599 private static class NumOps { 600 private final Object mLock = new Object(); 601 602 @GuardedBy("mLock") 603 private int[] mNumOps = new int[24]; 604 @GuardedBy("mLock") 605 private long mLastOpsHourSinceBoot; 606 607 /** 608 * Clear buckets of new hours that have elapsed since last operation. 609 * 610 * <p>I.e. when the last operation was triggered at 1:40 and the current operation was 611 * triggered at 4:03, the buckets "2, 3, and 4" are cleared. 612 * 613 * @param currentTime Current elapsed time since boot in ns 614 */ 615 void clearOldOps(long currentTime) { 616 synchronized (mLock) { 617 long numHoursSinceBoot = TimeUnit.HOURS.convert(currentTime, TimeUnit.NANOSECONDS); 618 619 // Clear buckets of new hours that have elapsed since last operation 620 // I.e. when the last operation was triggered at 1:40 and the current 621 // operation was triggered at 4:03, the bucket "2, 3, and 4" is cleared 622 if (mLastOpsHourSinceBoot != 0) { 623 for (long hour = mLastOpsHourSinceBoot + 1; hour <= numHoursSinceBoot; hour++) { 624 mNumOps[(int) (hour % 24)] = 0; 625 } 626 } 627 } 628 } 629 630 /** 631 * Add a new operation. 632 * 633 * @param currentTime Current elapsed time since boot in ns 634 */ 635 void addOp(long currentTime) { 636 synchronized (mLock) { 637 long numHoursSinceBoot = TimeUnit.HOURS.convert(currentTime, TimeUnit.NANOSECONDS); 638 639 mNumOps[(int) (numHoursSinceBoot % 24)]++; 640 mLastOpsHourSinceBoot = numHoursSinceBoot; 641 } 642 } 643 644 /** 645 * Get the total operations added in the last 24 hours. 646 * 647 * @return The total number of operations added in the last 24 hours 648 */ 649 int getOpsAdded() { 650 synchronized (mLock) { 651 int totalOperationsInLastDay = 0; 652 for (int i = 0; i < 24; i++) { 653 totalOperationsInLastDay += mNumOps[i]; 654 } 655 656 return totalOperationsInLastDay; 657 } 658 } 659 } 660 661 /** 662 * A single operation run in a {@link RemoteSoundTriggerDetectionService}. 663 * 664 * <p>Once the remote service is connected either setup + execute or setup + stop is executed. 665 */ 666 private static class Operation { 667 private interface ExecuteOp { 668 void run(int opId, ISoundTriggerDetectionService service) throws RemoteException; 669 } 670 671 private final @Nullable Runnable mSetupOp; 672 private final @NonNull ExecuteOp mExecuteOp; 673 private final @Nullable Runnable mDropOp; 674 675 private Operation(@Nullable Runnable setupOp, @NonNull ExecuteOp executeOp, 676 @Nullable Runnable cancelOp) { 677 mSetupOp = setupOp; 678 mExecuteOp = executeOp; 679 mDropOp = cancelOp; 680 } 681 682 private void setup() { 683 if (mSetupOp != null) { 684 mSetupOp.run(); 685 } 686 } 687 688 void run(int opId, @NonNull ISoundTriggerDetectionService service) throws RemoteException { 689 setup(); 690 mExecuteOp.run(opId, service); 691 } 692 693 void drop() { 694 setup(); 695 696 if (mDropOp != null) { 697 mDropOp.run(); 698 } 699 } 700 } 701 702 /** 703 * Local end for a {@link SoundTriggerDetectionService}. Operations are queued up and executed 704 * when the service connects. 705 * 706 * <p>If operations take too long they are forcefully aborted. 707 * 708 * <p>This also limits the amount of operations in 24 hours. 709 */ 710 private class RemoteSoundTriggerDetectionService 711 extends IRecognitionStatusCallback.Stub implements ServiceConnection { 712 private static final int MSG_STOP_ALL_PENDING_OPERATIONS = 1; 713 714 private final Object mRemoteServiceLock = new Object(); 715 716 /** UUID of the model the service is started for */ 717 private final @NonNull ParcelUuid mPuuid; 718 /** Params passed into the start method for the service */ 719 private final @Nullable Bundle mParams; 720 /** Component name passed when starting the service */ 721 private final @NonNull ComponentName mServiceName; 722 /** User that started the service */ 723 private final @NonNull UserHandle mUser; 724 /** Configuration of the recognition the service is handling */ 725 private final @NonNull RecognitionConfig mRecognitionConfig; 726 /** Wake lock keeping the remote service alive */ 727 private final @NonNull PowerManager.WakeLock mRemoteServiceWakeLock; 728 729 private final @NonNull Handler mHandler; 730 731 /** Callbacks that are called by the service */ 732 private final @NonNull ISoundTriggerDetectionServiceClient mClient; 733 734 /** Operations that are pending because the service is not yet connected */ 735 @GuardedBy("mRemoteServiceLock") 736 private final ArrayList<Operation> mPendingOps = new ArrayList<>(); 737 /** Operations that have been send to the service but have no yet finished */ 738 @GuardedBy("mRemoteServiceLock") 739 private final ArraySet<Integer> mRunningOpIds = new ArraySet<>(); 740 /** The number of operations executed in each of the last 24 hours */ 741 private final NumOps mNumOps; 742 743 /** The service binder if connected */ 744 @GuardedBy("mRemoteServiceLock") 745 private @Nullable ISoundTriggerDetectionService mService; 746 /** Whether the service has been bound */ 747 @GuardedBy("mRemoteServiceLock") 748 private boolean mIsBound; 749 /** Whether the service has been destroyed */ 750 @GuardedBy("mRemoteServiceLock") 751 private boolean mIsDestroyed; 752 /** 753 * Set once a final op is scheduled. No further ops can be added and the service is 754 * destroyed once the op finishes. 755 */ 756 @GuardedBy("mRemoteServiceLock") 757 private boolean mDestroyOnceRunningOpsDone; 758 759 /** Total number of operations performed by this service */ 760 @GuardedBy("mRemoteServiceLock") 761 private int mNumTotalOpsPerformed; 762 763 /** 764 * Create a new remote sound trigger detection service. This only binds to the service when 765 * operations are in flight. Each operation has a certain time it can run. Once no 766 * operations are allowed to run anymore, {@link #stopAllPendingOperations() all operations 767 * are aborted and stopped} and the service is disconnected. 768 * 769 * @param modelUuid The UUID of the model the recognition is for 770 * @param params The params passed to each method of the service 771 * @param serviceName The component name of the service 772 * @param user The user of the service 773 * @param config The configuration of the recognition 774 */ 775 public RemoteSoundTriggerDetectionService(@NonNull UUID modelUuid, 776 @Nullable Bundle params, @NonNull ComponentName serviceName, @NonNull UserHandle user, 777 @NonNull RecognitionConfig config) { 778 mPuuid = new ParcelUuid(modelUuid); 779 mParams = params; 780 mServiceName = serviceName; 781 mUser = user; 782 mRecognitionConfig = config; 783 mHandler = new Handler(Looper.getMainLooper()); 784 785 PowerManager pm = ((PowerManager) mContext.getSystemService(Context.POWER_SERVICE)); 786 mRemoteServiceWakeLock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, 787 "RemoteSoundTriggerDetectionService " + mServiceName.getPackageName() + ":" 788 + mServiceName.getClassName()); 789 790 synchronized (mLock) { 791 NumOps numOps = mNumOpsPerPackage.get(mServiceName.getPackageName()); 792 if (numOps == null) { 793 numOps = new NumOps(); 794 mNumOpsPerPackage.put(mServiceName.getPackageName(), numOps); 795 } 796 mNumOps = numOps; 797 } 798 799 mClient = new ISoundTriggerDetectionServiceClient.Stub() { 800 @Override 801 public void onOpFinished(int opId) { 802 long token = Binder.clearCallingIdentity(); 803 try { 804 synchronized (mRemoteServiceLock) { 805 mRunningOpIds.remove(opId); 806 807 if (mRunningOpIds.isEmpty() && mPendingOps.isEmpty()) { 808 if (mDestroyOnceRunningOpsDone) { 809 destroy(); 810 } else { 811 disconnectLocked(); 812 } 813 } 814 } 815 } finally { 816 Binder.restoreCallingIdentity(token); 817 } 818 } 819 }; 820 } 821 822 @Override 823 public boolean pingBinder() { 824 return !(mIsDestroyed || mDestroyOnceRunningOpsDone); 825 } 826 827 /** 828 * Disconnect from the service, but allow to re-connect when new operations are triggered. 829 */ 830 private void disconnectLocked() { 831 if (mService != null) { 832 try { 833 mService.removeClient(mPuuid); 834 } catch (Exception e) { 835 Slog.e(TAG, mPuuid + ": Cannot remove client", e); 836 } 837 838 mService = null; 839 } 840 841 if (mIsBound) { 842 mContext.unbindService(RemoteSoundTriggerDetectionService.this); 843 mIsBound = false; 844 845 synchronized (mCallbacksLock) { 846 mRemoteServiceWakeLock.release(); 847 } 848 } 849 } 850 851 /** 852 * Disconnect, do not allow to reconnect to the service. All further operations will be 853 * dropped. 854 */ 855 private void destroy() { 856 if (DEBUG) Slog.v(TAG, mPuuid + ": destroy"); 857 858 synchronized (mRemoteServiceLock) { 859 disconnectLocked(); 860 861 mIsDestroyed = true; 862 } 863 864 // The callback is removed before the flag is set 865 if (!mDestroyOnceRunningOpsDone) { 866 synchronized (mCallbacksLock) { 867 mCallbacks.remove(mPuuid.getUuid()); 868 } 869 } 870 } 871 872 /** 873 * Stop all pending operations and then disconnect for the service. 874 */ 875 private void stopAllPendingOperations() { 876 synchronized (mRemoteServiceLock) { 877 if (mIsDestroyed) { 878 return; 879 } 880 881 if (mService != null) { 882 int numOps = mRunningOpIds.size(); 883 for (int i = 0; i < numOps; i++) { 884 try { 885 mService.onStopOperation(mPuuid, mRunningOpIds.valueAt(i)); 886 } catch (Exception e) { 887 Slog.e(TAG, mPuuid + ": Could not stop operation " 888 + mRunningOpIds.valueAt(i), e); 889 } 890 } 891 892 mRunningOpIds.clear(); 893 } 894 895 disconnectLocked(); 896 } 897 } 898 899 /** 900 * Verify that the service has the expected properties and then bind to the service 901 */ 902 private void bind() { 903 long token = Binder.clearCallingIdentity(); 904 try { 905 Intent i = new Intent(); 906 i.setComponent(mServiceName); 907 908 ResolveInfo ri = mContext.getPackageManager().resolveServiceAsUser(i, 909 GET_SERVICES | GET_META_DATA | MATCH_DEBUG_TRIAGED_MISSING, 910 mUser.getIdentifier()); 911 912 if (ri == null) { 913 Slog.w(TAG, mPuuid + ": " + mServiceName + " not found"); 914 return; 915 } 916 917 if (!BIND_SOUND_TRIGGER_DETECTION_SERVICE 918 .equals(ri.serviceInfo.permission)) { 919 Slog.w(TAG, mPuuid + ": " + mServiceName + " does not require " 920 + BIND_SOUND_TRIGGER_DETECTION_SERVICE); 921 return; 922 } 923 924 mIsBound = mContext.bindServiceAsUser(i, this, 925 BIND_AUTO_CREATE | BIND_FOREGROUND_SERVICE, mUser); 926 927 if (mIsBound) { 928 mRemoteServiceWakeLock.acquire(); 929 } else { 930 Slog.w(TAG, mPuuid + ": Could not bind to " + mServiceName); 931 } 932 } finally { 933 Binder.restoreCallingIdentity(token); 934 } 935 } 936 937 /** 938 * Run an operation (i.e. send it do the service). If the service is not connected, this 939 * binds the service and then runs the operation once connected. 940 * 941 * @param op The operation to run 942 */ 943 private void runOrAddOperation(Operation op) { 944 synchronized (mRemoteServiceLock) { 945 if (mIsDestroyed || mDestroyOnceRunningOpsDone) { 946 Slog.w(TAG, mPuuid + ": Dropped operation as already destroyed or marked for " 947 + "destruction"); 948 949 op.drop(); 950 return; 951 } 952 953 if (mService == null) { 954 mPendingOps.add(op); 955 956 if (!mIsBound) { 957 bind(); 958 } 959 } else { 960 long currentTime = System.nanoTime(); 961 mNumOps.clearOldOps(currentTime); 962 963 // Drop operation if too many were executed in the last 24 hours. 964 int opsAllowed = Settings.Global.getInt(mContext.getContentResolver(), 965 MAX_SOUND_TRIGGER_DETECTION_SERVICE_OPS_PER_DAY, 966 Integer.MAX_VALUE); 967 968 // As we currently cannot dropping an op safely, disable throttling 969 int opsAdded = mNumOps.getOpsAdded(); 970 if (false && mNumOps.getOpsAdded() >= opsAllowed) { 971 try { 972 if (DEBUG || opsAllowed + 10 > opsAdded) { 973 Slog.w(TAG, mPuuid + ": Dropped operation as too many operations " 974 + "were run in last 24 hours"); 975 } 976 977 op.drop(); 978 } catch (Exception e) { 979 Slog.e(TAG, mPuuid + ": Could not drop operation", e); 980 } 981 } else { 982 mNumOps.addOp(currentTime); 983 984 // Find a free opID 985 int opId = mNumTotalOpsPerformed; 986 do { 987 mNumTotalOpsPerformed++; 988 } while (mRunningOpIds.contains(opId)); 989 990 // Run OP 991 try { 992 if (DEBUG) Slog.v(TAG, mPuuid + ": runOp " + opId); 993 994 op.run(opId, mService); 995 mRunningOpIds.add(opId); 996 } catch (Exception e) { 997 Slog.e(TAG, mPuuid + ": Could not run operation " + opId, e); 998 } 999 } 1000 1001 // Unbind from service if no operations are left (i.e. if the operation failed) 1002 if (mPendingOps.isEmpty() && mRunningOpIds.isEmpty()) { 1003 if (mDestroyOnceRunningOpsDone) { 1004 destroy(); 1005 } else { 1006 disconnectLocked(); 1007 } 1008 } else { 1009 mHandler.removeMessages(MSG_STOP_ALL_PENDING_OPERATIONS); 1010 mHandler.sendMessageDelayed(obtainMessage( 1011 RemoteSoundTriggerDetectionService::stopAllPendingOperations, this) 1012 .setWhat(MSG_STOP_ALL_PENDING_OPERATIONS), 1013 Settings.Global.getLong(mContext.getContentResolver(), 1014 SOUND_TRIGGER_DETECTION_SERVICE_OP_TIMEOUT, 1015 Long.MAX_VALUE)); 1016 } 1017 } 1018 } 1019 } 1020 1021 @Override 1022 public void onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent event) { 1023 Slog.w(TAG, mPuuid + "->" + mServiceName + ": IGNORED onKeyphraseDetected(" + event 1024 + ")"); 1025 } 1026 1027 /** 1028 * Create an AudioRecord enough for starting and releasing the data buffered for the event. 1029 * 1030 * @param event The event that was received 1031 * @return The initialized AudioRecord 1032 */ 1033 private @NonNull AudioRecord createAudioRecordForEvent( 1034 @NonNull SoundTrigger.GenericRecognitionEvent event) { 1035 AudioAttributes.Builder attributesBuilder = new AudioAttributes.Builder(); 1036 attributesBuilder.setInternalCapturePreset(MediaRecorder.AudioSource.HOTWORD); 1037 AudioAttributes attributes = attributesBuilder.build(); 1038 1039 // Use same AudioFormat processing as in RecognitionEvent.fromParcel 1040 AudioFormat originalFormat = event.getCaptureFormat(); 1041 AudioFormat captureFormat = (new AudioFormat.Builder()) 1042 .setChannelMask(originalFormat.getChannelMask()) 1043 .setEncoding(originalFormat.getEncoding()) 1044 .setSampleRate(originalFormat.getSampleRate()) 1045 .build(); 1046 1047 int bufferSize = AudioRecord.getMinBufferSize( 1048 captureFormat.getSampleRate() == AudioFormat.SAMPLE_RATE_UNSPECIFIED 1049 ? AudioFormat.SAMPLE_RATE_HZ_MAX 1050 : captureFormat.getSampleRate(), 1051 captureFormat.getChannelCount() == 2 1052 ? AudioFormat.CHANNEL_IN_STEREO 1053 : AudioFormat.CHANNEL_IN_MONO, 1054 captureFormat.getEncoding()); 1055 1056 return new AudioRecord(attributes, captureFormat, bufferSize, 1057 event.getCaptureSession()); 1058 } 1059 1060 @Override 1061 public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) { 1062 if (DEBUG) Slog.v(TAG, mPuuid + ": Generic sound trigger event: " + event); 1063 1064 runOrAddOperation(new Operation( 1065 // always execute: 1066 () -> { 1067 if (!mRecognitionConfig.allowMultipleTriggers) { 1068 // Unregister this remoteService once op is done 1069 synchronized (mCallbacksLock) { 1070 mCallbacks.remove(mPuuid.getUuid()); 1071 } 1072 mDestroyOnceRunningOpsDone = true; 1073 } 1074 }, 1075 // execute if not throttled: 1076 (opId, service) -> service.onGenericRecognitionEvent(mPuuid, opId, event), 1077 // execute if throttled: 1078 () -> { 1079 if (event.isCaptureAvailable()) { 1080 AudioRecord capturedData = createAudioRecordForEvent(event); 1081 1082 // Currently we need to start and release the audio record to reset 1083 // the DSP even if we don't want to process the event 1084 capturedData.startRecording(); 1085 capturedData.release(); 1086 } 1087 })); 1088 } 1089 1090 @Override 1091 public void onError(int status) { 1092 if (DEBUG) Slog.v(TAG, mPuuid + ": onError: " + status); 1093 1094 runOrAddOperation( 1095 new Operation( 1096 // always execute: 1097 () -> { 1098 // Unregister this remoteService once op is done 1099 synchronized (mCallbacksLock) { 1100 mCallbacks.remove(mPuuid.getUuid()); 1101 } 1102 mDestroyOnceRunningOpsDone = true; 1103 }, 1104 // execute if not throttled: 1105 (opId, service) -> service.onError(mPuuid, opId, status), 1106 // nothing to do if throttled 1107 null)); 1108 } 1109 1110 @Override 1111 public void onRecognitionPaused() { 1112 Slog.i(TAG, mPuuid + "->" + mServiceName + ": IGNORED onRecognitionPaused"); 1113 } 1114 1115 @Override 1116 public void onRecognitionResumed() { 1117 Slog.i(TAG, mPuuid + "->" + mServiceName + ": IGNORED onRecognitionResumed"); 1118 } 1119 1120 @Override 1121 public void onServiceConnected(ComponentName name, IBinder service) { 1122 if (DEBUG) Slog.v(TAG, mPuuid + ": onServiceConnected(" + service + ")"); 1123 1124 synchronized (mRemoteServiceLock) { 1125 mService = ISoundTriggerDetectionService.Stub.asInterface(service); 1126 1127 try { 1128 mService.setClient(mPuuid, mParams, mClient); 1129 } catch (Exception e) { 1130 Slog.e(TAG, mPuuid + ": Could not init " + mServiceName, e); 1131 return; 1132 } 1133 1134 while (!mPendingOps.isEmpty()) { 1135 runOrAddOperation(mPendingOps.remove(0)); 1136 } 1137 } 1138 } 1139 1140 @Override 1141 public void onServiceDisconnected(ComponentName name) { 1142 if (DEBUG) Slog.v(TAG, mPuuid + ": onServiceDisconnected"); 1143 1144 synchronized (mRemoteServiceLock) { 1145 mService = null; 1146 } 1147 } 1148 1149 @Override 1150 public void onBindingDied(ComponentName name) { 1151 if (DEBUG) Slog.v(TAG, mPuuid + ": onBindingDied"); 1152 1153 synchronized (mRemoteServiceLock) { 1154 destroy(); 1155 } 1156 } 1157 1158 @Override 1159 public void onNullBinding(ComponentName name) { 1160 Slog.w(TAG, name + " for model " + mPuuid + " returned a null binding"); 1161 1162 synchronized (mRemoteServiceLock) { 1163 disconnectLocked(); 1164 } 1165 } 1166 } 1167 1168 private void grabWakeLock() { 1169 synchronized (mCallbacksLock) { 1170 if (mWakelock == null) { 1171 PowerManager pm = ((PowerManager) mContext.getSystemService(Context.POWER_SERVICE)); 1172 mWakelock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, TAG); 1173 } 1174 mWakelock.acquire(); 1175 } 1176 } 1177 1178 private PendingIntent.OnFinished mCallbackCompletedHandler = new PendingIntent.OnFinished() { 1179 @Override 1180 public void onSendFinished(PendingIntent pendingIntent, Intent intent, int resultCode, 1181 String resultData, Bundle resultExtras) { 1182 // We're only ever invoked when the callback is done, so release the lock. 1183 synchronized (mCallbacksLock) { 1184 mWakelock.release(); 1185 } 1186 } 1187 }; 1188 1189 public final class LocalSoundTriggerService extends SoundTriggerInternal { 1190 private final Context mContext; 1191 private SoundTriggerHelper mSoundTriggerHelper; 1192 1193 LocalSoundTriggerService(Context context) { 1194 mContext = context; 1195 } 1196 1197 synchronized void setSoundTriggerHelper(SoundTriggerHelper helper) { 1198 mSoundTriggerHelper = helper; 1199 } 1200 1201 @Override 1202 public int startRecognition(int keyphraseId, KeyphraseSoundModel soundModel, 1203 IRecognitionStatusCallback listener, RecognitionConfig recognitionConfig) { 1204 if (!isInitialized()) return STATUS_ERROR; 1205 return mSoundTriggerHelper.startKeyphraseRecognition(keyphraseId, soundModel, listener, 1206 recognitionConfig); 1207 } 1208 1209 @Override 1210 public synchronized int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { 1211 if (!isInitialized()) return STATUS_ERROR; 1212 return mSoundTriggerHelper.stopKeyphraseRecognition(keyphraseId, listener); 1213 } 1214 1215 @Override 1216 public ModuleProperties getModuleProperties() { 1217 if (!isInitialized()) return null; 1218 return mSoundTriggerHelper.getModuleProperties(); 1219 } 1220 1221 @Override 1222 public int unloadKeyphraseModel(int keyphraseId) { 1223 if (!isInitialized()) return STATUS_ERROR; 1224 return mSoundTriggerHelper.unloadKeyphraseSoundModel(keyphraseId); 1225 } 1226 1227 @Override 1228 public void dump(FileDescriptor fd, PrintWriter pw, String[] args) { 1229 if (!isInitialized()) return; 1230 mSoundTriggerHelper.dump(fd, pw, args); 1231 } 1232 1233 private synchronized boolean isInitialized() { 1234 if (mSoundTriggerHelper == null ) { 1235 Slog.e(TAG, "SoundTriggerHelper not initialized."); 1236 return false; 1237 } 1238 return true; 1239 } 1240 } 1241 1242 private void enforceCallingPermission(String permission) { 1243 if (mContext.checkCallingOrSelfPermission(permission) 1244 != PackageManager.PERMISSION_GRANTED) { 1245 throw new SecurityException("Caller does not hold the permission " + permission); 1246 } 1247 } 1248 } 1249
