Home | History | Annotate | Download | only in speech 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/speech/speech_recognition_bubble.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/lazy_instance.h"
      9 #include "base/message_loop/message_loop.h"
     10 #include "content/public/browser/web_contents.h"
     11 #include "content/public/browser/web_contents_view.h"
     12 #include "grit/generated_resources.h"
     13 #include "grit/theme_resources.h"
     14 #include "ui/base/resource/resource_bundle.h"
     15 #include "ui/gfx/canvas.h"
     16 #include "ui/gfx/display.h"
     17 #include "ui/gfx/image/image_skia_operations.h"
     18 #include "ui/gfx/rect.h"
     19 #include "ui/gfx/screen.h"
     20 
     21 using content::WebContents;
     22 
     23 namespace {
     24 
     25 const color_utils::HSL kGrayscaleShift = { -1, 0, 0.6 };
     26 const int kWarmingUpAnimationStartMs = 500;
     27 const int kWarmingUpAnimationStepMs = 100;
     28 const int kRecognizingAnimationStepMs = 100;
     29 
     30 // A lazily initialized singleton to hold all the image used by the speech
     31 // recognition bubbles and safely destroy them on exit.
     32 class SpeechRecognitionBubbleImages {
     33  public:
     34   const std::vector<gfx::ImageSkia>& spinner() const { return spinner_; }
     35   const std::vector<gfx::ImageSkia>& warm_up() const { return warm_up_; }
     36   gfx::ImageSkia* mic_full() const { return mic_full_; }
     37   gfx::ImageSkia* mic_empty() const { return mic_empty_; }
     38   gfx::ImageSkia* mic_noise() const { return mic_noise_; }
     39   gfx::ImageSkia* mic_mask() const { return mic_mask_; }
     40 
     41  private:
     42   // Private constructor to enforce singleton.
     43   friend struct base::DefaultLazyInstanceTraits<SpeechRecognitionBubbleImages>;
     44   SpeechRecognitionBubbleImages();
     45 
     46   std::vector<gfx::ImageSkia> spinner_;  // Frames for the progress spinner.
     47   std::vector<gfx::ImageSkia> warm_up_;  // Frames for the warm up animation.
     48 
     49   // These images are owned by ResourceBundle and need not be destroyed.
     50   gfx::ImageSkia* mic_full_;  // Mic image with full volume.
     51   gfx::ImageSkia* mic_noise_;  // Mic image with full noise volume.
     52   gfx::ImageSkia* mic_empty_;  // Mic image with zero volume.
     53   gfx::ImageSkia* mic_mask_;  // Gradient mask used by the volume indicator.
     54 };
     55 
     56 SpeechRecognitionBubbleImages::SpeechRecognitionBubbleImages() {
     57   mic_empty_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
     58       IDR_SPEECH_INPUT_MIC_EMPTY);
     59   mic_noise_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
     60       IDR_SPEECH_INPUT_MIC_NOISE);
     61   mic_full_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
     62       IDR_SPEECH_INPUT_MIC_FULL);
     63   mic_mask_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
     64       IDR_SPEECH_INPUT_MIC_MASK);
     65 
     66   // The sprite image consists of all the animation frames put together in one
     67   // horizontal/wide image. Each animation frame is square in shape within the
     68   // sprite.
     69   const gfx::ImageSkia* spinner_image = ui::ResourceBundle::GetSharedInstance().
     70       GetImageSkiaNamed(IDR_SPEECH_INPUT_SPINNER);
     71   int frame_size = spinner_image->height();
     72 
     73   // When recording starts up, it may take a short while (few ms or even a
     74   // couple of seconds) before the audio device starts really capturing data.
     75   // This is more apparent on first use. To cover such cases we show a warming
     76   // up state in the bubble starting with a blank spinner image. If audio data
     77   // starts coming in within a couple hundred ms, we switch to the recording
     78   // UI and if it takes longer, we show the real warm up animation frames.
     79   // This reduces visual jank for the most part.
     80   SkBitmap empty_spinner;
     81   empty_spinner.setConfig(SkBitmap::kARGB_8888_Config, frame_size, frame_size);
     82   empty_spinner.allocPixels();
     83   empty_spinner.eraseRGB(255, 255, 255);
     84   // |empty_spinner| has solid color. Pixel doubling a solid color is ok.
     85   warm_up_.push_back(gfx::ImageSkia::CreateFrom1xBitmap(empty_spinner));
     86 
     87   for (gfx::Rect src_rect(frame_size, frame_size);
     88        src_rect.x() < spinner_image->width();
     89        src_rect.Offset(frame_size, 0)) {
     90     gfx::ImageSkia frame = gfx::ImageSkiaOperations::ExtractSubset(
     91         *spinner_image, src_rect);
     92 
     93     // The image created by ExtractSubset just points to the same pixels as
     94     // the original and adjusts rowBytes accordingly. However that doesn't
     95     // render properly and gets vertically squished in Linux due to a bug in
     96     // Skia. Until that gets fixed we work around by taking a real copy of it
     97     // below as the copied image has the correct rowBytes and renders fine.
     98     frame.EnsureRepsForSupportedScaleFactors();
     99     std::vector<gfx::ImageSkiaRep> image_reps = frame.image_reps();
    100     gfx::ImageSkia frame_copy;
    101     for (size_t i = 0; i < image_reps.size(); ++i) {
    102       const SkBitmap& copy_src = image_reps[i].sk_bitmap();
    103       SkBitmap copy_dst;
    104       copy_src.copyTo(&copy_dst, SkBitmap::kARGB_8888_Config);
    105       frame_copy.AddRepresentation(gfx::ImageSkiaRep(
    106           copy_dst, image_reps[i].scale_factor()));
    107     }
    108     spinner_.push_back(frame_copy);
    109 
    110     // The warm up spinner animation is a gray scale version of the real one.
    111     warm_up_.push_back(gfx::ImageSkiaOperations::CreateHSLShiftedImage(
    112         frame_copy, kGrayscaleShift));
    113   }
    114 }
    115 
    116 base::LazyInstance<SpeechRecognitionBubbleImages> g_images =
    117     LAZY_INSTANCE_INITIALIZER;
    118 
    119 }  // namespace
    120 
    121 SpeechRecognitionBubble::FactoryMethod SpeechRecognitionBubble::factory_ = NULL;
    122 const int SpeechRecognitionBubble::kBubbleTargetOffsetX = 10;
    123 
    124 SpeechRecognitionBubble* SpeechRecognitionBubble::Create(
    125     WebContents* web_contents, Delegate* delegate,
    126     const gfx::Rect& element_rect) {
    127   if (factory_)
    128     return (*factory_)(web_contents, delegate, element_rect);
    129 
    130   // Has the tab already closed before bubble create request was processed?
    131   if (!web_contents)
    132     return NULL;
    133 
    134   return CreateNativeBubble(web_contents, delegate, element_rect);
    135 }
    136 
    137 SpeechRecognitionBubbleBase::SpeechRecognitionBubbleBase(
    138     WebContents* web_contents)
    139     : weak_factory_(this),
    140       animation_step_(0),
    141       display_mode_(DISPLAY_MODE_RECORDING),
    142       web_contents_(web_contents),
    143       scale_factor_(ui::SCALE_FACTOR_NONE) {
    144   gfx::NativeView view =
    145       web_contents_ ? web_contents_->GetView()->GetNativeView() : NULL;
    146   gfx::Screen* screen = gfx::Screen::GetScreenFor(view);
    147   gfx::Display display = screen->GetDisplayNearestWindow(view);
    148   scale_factor_ = ui::GetScaleFactorFromScale(
    149       display.device_scale_factor());
    150 
    151   const gfx::ImageSkiaRep& rep =
    152       g_images.Get().mic_empty()->GetRepresentation(scale_factor_);
    153   mic_image_.reset(new SkBitmap());
    154   mic_image_->setConfig(SkBitmap::kARGB_8888_Config,
    155       rep.pixel_width(), rep.pixel_height());
    156   mic_image_->allocPixels();
    157 
    158   buffer_image_.reset(new SkBitmap());
    159   buffer_image_->setConfig(SkBitmap::kARGB_8888_Config,
    160       rep.pixel_width(), rep.pixel_height());
    161   buffer_image_->allocPixels();
    162 }
    163 
    164 SpeechRecognitionBubbleBase::~SpeechRecognitionBubbleBase() {
    165   // This destructor is added to make sure members such as the scoped_ptr
    166   // get destroyed here and the derived classes don't have to care about such
    167   // member variables which they don't use.
    168 }
    169 
    170 void SpeechRecognitionBubbleBase::SetWarmUpMode() {
    171   weak_factory_.InvalidateWeakPtrs();
    172   display_mode_ = DISPLAY_MODE_WARM_UP;
    173   animation_step_ = 0;
    174   DoWarmingUpAnimationStep();
    175   UpdateLayout();
    176 }
    177 
    178 void SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep() {
    179   SetImage(g_images.Get().warm_up()[animation_step_]);
    180   base::MessageLoop::current()->PostDelayedTask(
    181       FROM_HERE,
    182       base::Bind(&SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep,
    183           weak_factory_.GetWeakPtr()),
    184       base::TimeDelta::FromMilliseconds(
    185           animation_step_ == 0 ? kWarmingUpAnimationStartMs
    186                                : kWarmingUpAnimationStepMs));
    187   if (++animation_step_ >= static_cast<int>(g_images.Get().warm_up().size()))
    188     animation_step_ = 1;  // Frame 0 is skipped during the animation.
    189 }
    190 
    191 void SpeechRecognitionBubbleBase::SetRecordingMode() {
    192   weak_factory_.InvalidateWeakPtrs();
    193   display_mode_ = DISPLAY_MODE_RECORDING;
    194   SetInputVolume(0, 0);
    195   UpdateLayout();
    196 }
    197 
    198 void SpeechRecognitionBubbleBase::SetRecognizingMode() {
    199   display_mode_ = DISPLAY_MODE_RECOGNIZING;
    200   animation_step_ = 0;
    201   DoRecognizingAnimationStep();
    202   UpdateLayout();
    203 }
    204 
    205 void SpeechRecognitionBubbleBase::DoRecognizingAnimationStep() {
    206   SetImage(g_images.Get().spinner()[animation_step_]);
    207   if (++animation_step_ >= static_cast<int>(g_images.Get().spinner().size()))
    208     animation_step_ = 0;
    209   base::MessageLoop::current()->PostDelayedTask(
    210       FROM_HERE,
    211       base::Bind(&SpeechRecognitionBubbleBase::DoRecognizingAnimationStep,
    212           weak_factory_.GetWeakPtr()),
    213       base::TimeDelta::FromMilliseconds(kRecognizingAnimationStepMs));
    214 }
    215 
    216 void SpeechRecognitionBubbleBase::SetMessage(const string16& text) {
    217   weak_factory_.InvalidateWeakPtrs();
    218   message_text_ = text;
    219   display_mode_ = DISPLAY_MODE_MESSAGE;
    220   UpdateLayout();
    221 }
    222 
    223 void SpeechRecognitionBubbleBase::DrawVolumeOverlay(SkCanvas* canvas,
    224                                                     const gfx::ImageSkia& image,
    225                                                     float volume) {
    226   buffer_image_->eraseARGB(0, 0, 0, 0);
    227 
    228   int width = mic_image_->width();
    229   int height = mic_image_->height();
    230   SkCanvas buffer_canvas(*buffer_image_);
    231 
    232   buffer_canvas.save();
    233   const int kVolumeSteps = 12;
    234   SkScalar clip_right =
    235       (((1.0f - volume) * (width * (kVolumeSteps + 1))) - width) / kVolumeSteps;
    236   buffer_canvas.clipRect(SkRect::MakeLTRB(0, 0,
    237       SkIntToScalar(width) - clip_right, SkIntToScalar(height)));
    238   buffer_canvas.drawBitmap(
    239       image.GetRepresentation(scale_factor_).sk_bitmap(), 0, 0);
    240   buffer_canvas.restore();
    241   SkPaint multiply_paint;
    242   multiply_paint.setXfermodeMode(SkXfermode::kModulate_Mode);
    243   buffer_canvas.drawBitmap(
    244       g_images.Get().mic_mask()->GetRepresentation(scale_factor_).sk_bitmap(),
    245       -clip_right, 0, &multiply_paint);
    246 
    247   canvas->drawBitmap(*buffer_image_.get(), 0, 0);
    248 }
    249 
    250 void SpeechRecognitionBubbleBase::SetInputVolume(float volume,
    251                                                  float noise_volume) {
    252   mic_image_->eraseARGB(0, 0, 0, 0);
    253   SkCanvas canvas(*mic_image_);
    254 
    255   // Draw the empty volume image first and the current volume image on top,
    256   // and then the noise volume image on top of both.
    257   canvas.drawBitmap(
    258       g_images.Get().mic_empty()->GetRepresentation(scale_factor_).sk_bitmap(),
    259       0, 0);
    260   DrawVolumeOverlay(&canvas, *g_images.Get().mic_full(), volume);
    261   DrawVolumeOverlay(&canvas, *g_images.Get().mic_noise(), noise_volume);
    262 
    263   gfx::ImageSkia image(gfx::ImageSkiaRep(*mic_image_.get(), scale_factor_));
    264   SetImage(image);
    265 }
    266 
    267 WebContents* SpeechRecognitionBubbleBase::GetWebContents() {
    268   return web_contents_;
    269 }
    270 
    271 void SpeechRecognitionBubbleBase::SetImage(const gfx::ImageSkia& image) {
    272   icon_image_ = image;
    273   UpdateImage();
    274 }
    275 
    276 gfx::ImageSkia SpeechRecognitionBubbleBase::icon_image() {
    277   return icon_image_;
    278 }
    279