1 /* 2 * Copyright (C) 2009 Google Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 * use this file except in compliance with the License. You may obtain a copy of 6 * the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 * License for the specific language governing permissions and limitations under 14 * the License. 15 */ 16 17 package com.android.inputmethod.voice; 18 19 import java.io.ByteArrayOutputStream; 20 import java.nio.ByteBuffer; 21 import java.nio.ByteOrder; 22 import java.nio.ShortBuffer; 23 import java.util.ArrayList; 24 import java.util.List; 25 26 import android.content.ContentResolver; 27 import android.content.Context; 28 import android.content.res.Resources; 29 import android.graphics.Bitmap; 30 import android.graphics.Canvas; 31 import android.graphics.CornerPathEffect; 32 import android.graphics.Paint; 33 import android.graphics.Path; 34 import android.graphics.PathEffect; 35 import android.graphics.drawable.Drawable; 36 import android.os.Handler; 37 import android.util.TypedValue; 38 import android.view.LayoutInflater; 39 import android.view.View; 40 import android.view.View.OnClickListener; 41 import android.view.ViewGroup.MarginLayoutParams; 42 import android.widget.ImageView; 43 import android.widget.ProgressBar; 44 import android.widget.TextView; 45 46 import com.android.inputmethod.latin.R; 47 48 /** 49 * The user interface for the "Speak now" and "working" states. 50 * Displays a recognition dialog (with waveform, voice meter, etc.), 51 * plays beeps, shows errors, etc. 52 */ 53 public class RecognitionView { 54 private static final String TAG = "RecognitionView"; 55 56 private Handler mUiHandler; // Reference to UI thread 57 private View mView; 58 private Context mContext; 59 60 private ImageView mImage; 61 private TextView mText; 62 private View mButton; 63 private TextView mButtonText; 64 private View mProgress; 65 66 private Drawable mInitializing; 67 private Drawable mError; 68 private List<Drawable> mSpeakNow; 69 70 private float mVolume = 0.0f; 71 private int mLevel = 0; 72 73 private enum State {LISTENING, WORKING, READY} 74 private State mState = State.READY; 75 76 private float mMinMicrophoneLevel; 77 private float mMaxMicrophoneLevel; 78 79 /** Updates the microphone icon to show user their volume.*/ 80 private Runnable mUpdateVolumeRunnable = new Runnable() { 81 public void run() { 82 if (mState != State.LISTENING) { 83 return; 84 } 85 86 final float min = mMinMicrophoneLevel; 87 final float max = mMaxMicrophoneLevel; 88 final int maxLevel = mSpeakNow.size() - 1; 89 90 int index = (int) ((mVolume - min) / (max - min) * maxLevel); 91 final int level = Math.min(Math.max(0, index), maxLevel); 92 93 if (level != mLevel) { 94 mImage.setImageDrawable(mSpeakNow.get(level)); 95 mLevel = level; 96 } 97 mUiHandler.postDelayed(mUpdateVolumeRunnable, 50); 98 } 99 }; 100 101 public RecognitionView(Context context, OnClickListener clickListener) { 102 mUiHandler = new Handler(); 103 104 LayoutInflater inflater = (LayoutInflater) context.getSystemService( 105 Context.LAYOUT_INFLATER_SERVICE); 106 mView = inflater.inflate(R.layout.recognition_status, null); 107 ContentResolver cr = context.getContentResolver(); 108 mMinMicrophoneLevel = SettingsUtil.getSettingsFloat( 109 cr, SettingsUtil.LATIN_IME_MIN_MICROPHONE_LEVEL, 15.f); 110 mMaxMicrophoneLevel = SettingsUtil.getSettingsFloat( 111 cr, SettingsUtil.LATIN_IME_MAX_MICROPHONE_LEVEL, 30.f); 112 113 // Pre-load volume level images 114 Resources r = context.getResources(); 115 116 mSpeakNow = new ArrayList<Drawable>(); 117 mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level0)); 118 mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level1)); 119 mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level2)); 120 mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level3)); 121 mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level4)); 122 mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level5)); 123 mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level6)); 124 125 mInitializing = r.getDrawable(R.drawable.mic_slash); 126 mError = r.getDrawable(R.drawable.caution); 127 128 mImage = (ImageView) mView.findViewById(R.id.image); 129 mButton = mView.findViewById(R.id.button); 130 mButton.setOnClickListener(clickListener); 131 mText = (TextView) mView.findViewById(R.id.text); 132 mButtonText = (TextView) mView.findViewById(R.id.button_text); 133 mProgress = mView.findViewById(R.id.progress); 134 135 mContext = context; 136 } 137 138 public View getView() { 139 return mView; 140 } 141 142 public void restoreState() { 143 mUiHandler.post(new Runnable() { 144 public void run() { 145 // Restart the spinner 146 if (mState == State.WORKING) { 147 ((ProgressBar)mProgress).setIndeterminate(false); 148 ((ProgressBar)mProgress).setIndeterminate(true); 149 } 150 } 151 }); 152 } 153 154 public void showInitializing() { 155 mUiHandler.post(new Runnable() { 156 public void run() { 157 prepareDialog(false, mContext.getText(R.string.voice_initializing), mInitializing, 158 mContext.getText(R.string.cancel)); 159 } 160 }); 161 } 162 163 public void showListening() { 164 mUiHandler.post(new Runnable() { 165 public void run() { 166 mState = State.LISTENING; 167 prepareDialog(false, mContext.getText(R.string.voice_listening), mSpeakNow.get(0), 168 mContext.getText(R.string.cancel)); 169 } 170 }); 171 mUiHandler.postDelayed(mUpdateVolumeRunnable, 50); 172 } 173 174 public void updateVoiceMeter(final float rmsdB) { 175 mVolume = rmsdB; 176 } 177 178 public void showError(final String message) { 179 mUiHandler.post(new Runnable() { 180 public void run() { 181 mState = State.READY; 182 prepareDialog(false, message, mError, mContext.getText(R.string.ok)); 183 } 184 }); 185 } 186 187 public void showWorking( 188 final ByteArrayOutputStream waveBuffer, 189 final int speechStartPosition, 190 final int speechEndPosition) { 191 192 mUiHandler.post(new Runnable() { 193 public void run() { 194 mState = State.WORKING; 195 prepareDialog(true, mContext.getText(R.string.voice_working), null, mContext 196 .getText(R.string.cancel)); 197 final ShortBuffer buf = ByteBuffer.wrap(waveBuffer.toByteArray()).order( 198 ByteOrder.nativeOrder()).asShortBuffer(); 199 buf.position(0); 200 waveBuffer.reset(); 201 showWave(buf, speechStartPosition / 2, speechEndPosition / 2); 202 } 203 }); 204 } 205 206 private void prepareDialog(boolean spinVisible, CharSequence text, Drawable image, 207 CharSequence btnTxt) { 208 if (spinVisible) { 209 mProgress.setVisibility(View.VISIBLE); 210 mImage.setVisibility(View.GONE); 211 } else { 212 mProgress.setVisibility(View.GONE); 213 mImage.setImageDrawable(image); 214 mImage.setVisibility(View.VISIBLE); 215 } 216 mText.setText(text); 217 mButtonText.setText(btnTxt); 218 } 219 220 /** 221 * @return an average abs of the specified buffer. 222 */ 223 private static int getAverageAbs(ShortBuffer buffer, int start, int i, int npw) { 224 int from = start + i * npw; 225 int end = from + npw; 226 int total = 0; 227 for (int x = from; x < end; x++) { 228 total += Math.abs(buffer.get(x)); 229 } 230 return total / npw; 231 } 232 233 234 /** 235 * Shows waveform of input audio. 236 * 237 * Copied from version in VoiceSearch's RecognitionActivity. 238 * 239 * TODO: adjust stroke width based on the size of data. 240 * TODO: use dip rather than pixels. 241 */ 242 private void showWave(ShortBuffer waveBuffer, int startPosition, int endPosition) { 243 final int w = ((View) mImage.getParent()).getWidth(); 244 final int h = mImage.getHeight(); 245 if (w <= 0 || h <= 0) { 246 // view is not visible this time. Skip drawing. 247 return; 248 } 249 final Bitmap b = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); 250 final Canvas c = new Canvas(b); 251 final Paint paint = new Paint(); 252 paint.setColor(0xFFFFFFFF); // 0xAARRGGBB 253 paint.setAntiAlias(true); 254 paint.setStyle(Paint.Style.STROKE); 255 paint.setAlpha(0x90); 256 257 final PathEffect effect = new CornerPathEffect(3); 258 paint.setPathEffect(effect); 259 260 final int numSamples = waveBuffer.remaining(); 261 int endIndex; 262 if (endPosition == 0) { 263 endIndex = numSamples; 264 } else { 265 endIndex = Math.min(endPosition, numSamples); 266 } 267 268 int startIndex = startPosition - 2000; // include 250ms before speech 269 if (startIndex < 0) { 270 startIndex = 0; 271 } 272 final int numSamplePerWave = 200; // 8KHz 25ms = 200 samples 273 final float scale = 10.0f / 65536.0f; 274 275 final int count = (endIndex - startIndex) / numSamplePerWave; 276 final float deltaX = 1.0f * w / count; 277 int yMax = h / 2 - 8; 278 Path path = new Path(); 279 c.translate(0, yMax); 280 float x = 0; 281 path.moveTo(x, 0); 282 for (int i = 0; i < count; i++) { 283 final int avabs = getAverageAbs(waveBuffer, startIndex, i , numSamplePerWave); 284 int sign = ( (i & 01) == 0) ? -1 : 1; 285 final float y = Math.min(yMax, avabs * h * scale) * sign; 286 path.lineTo(x, y); 287 x += deltaX; 288 path.lineTo(x, y); 289 } 290 if (deltaX > 4) { 291 paint.setStrokeWidth(3); 292 } else { 293 paint.setStrokeWidth(Math.max(1, (int) (deltaX -.05))); 294 } 295 c.drawPath(path, paint); 296 mImage.setImageBitmap(b); 297 mImage.setVisibility(View.VISIBLE); 298 MarginLayoutParams mProgressParams = (MarginLayoutParams)mProgress.getLayoutParams(); 299 mProgressParams.topMargin = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_PX, 300 -h , mContext.getResources().getDisplayMetrics()); 301 302 // Tweak the padding manually to fill out the whole view horizontally. 303 // TODO: Do this in the xml layout instead. 304 ((View) mImage.getParent()).setPadding(4, ((View) mImage.getParent()).getPaddingTop(), 3, 305 ((View) mImage.getParent()).getPaddingBottom()); 306 mProgress.setLayoutParams(mProgressParams); 307 } 308 309 310 public void finish() { 311 mUiHandler.post(new Runnable() { 312 public void run() { 313 mState = State.READY; 314 exitWorking(); 315 } 316 }); 317 } 318 319 private void exitWorking() { 320 mProgress.setVisibility(View.GONE); 321 mImage.setVisibility(View.VISIBLE); 322 } 323 } 324
