Home | History | Annotate | Download | only in voicedialer 
      1 /*
      2  * Copyright (C) 2007 Google Inc.
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.voicedialer;
     18 
     19 import android.content.Intent;
     20 import android.util.Config;
     21 import android.util.Log;
     22 import java.io.File;
     23 import java.io.FileFilter;
     24 import java.util.Arrays;
     25 import java.util.HashSet;
     26 import java.util.Set;
     27 import java.util.Vector;
     28 
     29 /**
     30  * This class represents a person who may be called via the VoiceDialer app.
     31  * The person has a name and a list of phones (home, mobile, work).
     32  */
     33 public class VoiceDialerTester {
     34     private static final String TAG = "VoiceDialerTester";
     35 
     36     private final WavFile[] mWavFiles;
     37     private final File[] mWavDirs;
     38 
     39     // these indicate the current test
     40     private int mWavFile = -1; // -1 so it will step to 0 on first iteration
     41 
     42     private static class WavFile {
     43         final public File mFile;
     44         public int mRank;
     45         public int mTotal;
     46         public String mMessage;
     47 
     48         public WavFile(File file) {
     49             mFile = file;
     50         }
     51     }
     52 
     53     /**
     54      * Sweep directory of directories, listing all WAV files.
     55      */
     56     public VoiceDialerTester(File dir) {
     57         if (Config.LOGD) {
     58             Log.d(TAG, "VoiceDialerTester " + dir);
     59         }
     60 
     61         // keep a list of directories visited
     62         Vector<File> wavDirs = new Vector<File>();
     63         wavDirs.add(dir);
     64 
     65         // scan the directory tree
     66         Vector<File> wavFiles = new Vector<File>();
     67         for (int i = 0; i < wavDirs.size(); i++) {
     68             File d = wavDirs.get(i);
     69             for (File f : d.listFiles()) {
     70                 if (f.isFile() && f.getName().endsWith(".wav")) {
     71                     wavFiles.add(f);
     72                 }
     73                 else if (f.isDirectory()) {
     74                     wavDirs.add(f);
     75                 }
     76             }
     77         }
     78 
     79         // produce a sorted list of WavFiles
     80         File[] fa = wavFiles.toArray(new File[wavFiles.size()]);
     81         Arrays.sort(fa);
     82         mWavFiles = new WavFile[fa.length];
     83         for (int i = 0; i < mWavFiles.length; i++) {
     84             mWavFiles[i] = new WavFile(fa[i]);
     85         }
     86 
     87         // produce a sorted list of directories
     88         mWavDirs = wavDirs.toArray(new File[wavDirs.size()]);
     89         Arrays.sort(mWavDirs);
     90     }
     91 
     92     public File getWavFile() {
     93         return mWavFiles[mWavFile].mFile;
     94     }
     95 
     96     /**
     97      * Called by VoiceDialerActivity when a recognizer error occurs.
     98      */
     99     public void onRecognitionError(String msg) {
    100         WavFile wf = mWavFiles[mWavFile];
    101         wf.mRank = -1;
    102         wf.mTotal = -1;
    103         wf.mMessage = msg;
    104     }
    105 
    106     /**
    107      * Called by VoiceDialerActivity when a recognizer failure occurs.
    108      * @param msg Message to display.
    109      */
    110     public void onRecognitionFailure(String msg) {
    111         WavFile wf = mWavFiles[mWavFile];
    112         wf.mRank = 0;
    113         wf.mTotal = 0;
    114         wf.mMessage = msg;
    115     }
    116 
    117     /**
    118      * Called by VoiceDialerActivity when the recognizer succeeds.
    119      * @param intents Array of Intents corresponding to recognized sentences.
    120      */
    121     public void onRecognitionSuccess(Intent[] intents) {
    122         WavFile wf = mWavFiles[mWavFile];
    123         wf.mTotal = intents.length;
    124         String utter = wf.mFile.getName().toLowerCase().replace('_', ' ');
    125         utter = utter.substring(0, utter.indexOf('.')).trim();
    126         for (int i = 0; i < intents.length; i++) {
    127             String sentence =
    128                     intents[i].getStringExtra(RecognizerEngine.SENTENCE_EXTRA).
    129                     toLowerCase().trim();
    130             // note the first in case there are no matches
    131             if (i == 0) {
    132                 wf.mMessage = sentence;
    133                 if (intents.length > 1) wf.mMessage += ", etc";
    134             }
    135             // is this a match
    136             if (utter.equals(sentence)) {
    137                 wf.mRank = i + 1;
    138                 wf.mMessage = null;
    139                 break;
    140             }
    141         }
    142     }
    143 
    144     /**
    145      * Called to step to the next WAV file in the test set.
    146      * @return true if successful, false if no more test files.
    147      */
    148     public boolean stepToNextTest() {
    149         mWavFile++;
    150         return mWavFile < mWavFiles.length;
    151     }
    152 
    153     private static final String REPORT_FMT = "%6s %6s %6s %6s %6s %6s %6s %s";
    154     private static final String REPORT_HDR = String.format(REPORT_FMT,
    155             "1/1", "1/N", "M/N", "0/N", "Fail", "Error", "Total", "");
    156 
    157     /**
    158      * Called when the test is complete to dump a summary.
    159      */
    160     public void report() {
    161         // report for each file
    162         Log.d(TAG, "List of all utterances tested");
    163         for (WavFile wf : mWavFiles) {
    164             Log.d(TAG, wf.mRank + "/" + wf.mTotal + "  " + wf.mFile +
    165                     (wf.mMessage != null ? "  " + wf.mMessage : ""));
    166         }
    167 
    168         // summary reports by file name
    169         reportSummaryForEachFileName();
    170 
    171         // summary reports by directory name
    172         reportSummaryForEachDir();
    173 
    174         // summary report for all files
    175         Log.d(TAG, "Summary of all utterances");
    176         Log.d(TAG, REPORT_HDR);
    177         reportSummary("Total", null);
    178     }
    179 
    180     private void reportSummaryForEachFileName() {
    181         Set<String> set = new HashSet<String>();
    182         for (WavFile wf : mWavFiles) {
    183             set.add(wf.mFile.getName());
    184         }
    185         String[] names = set.toArray(new String[set.size()]);
    186         Arrays.sort(names);
    187 
    188         Log.d(TAG, "Summary of utternaces by filename");
    189         Log.d(TAG, REPORT_HDR);
    190         for (final String fn : names) {
    191             reportSummary(fn,
    192                     new FileFilter() {
    193                         public boolean accept(File file) {
    194                             return fn.equals(file.getName());
    195                         }
    196             });
    197         }
    198     }
    199 
    200     private void reportSummaryForEachDir() {
    201         Set<String> set = new HashSet<String>();
    202         for (WavFile wf : mWavFiles) {
    203             set.add(wf.mFile.getParent());
    204         }
    205         String[] names = set.toArray(new String[set.size()]);
    206         Arrays.sort(names);
    207 
    208         Log.d(TAG, "Summary of utterances by directory");
    209         Log.d(TAG, REPORT_HDR);
    210         for (File dir : mWavDirs) {
    211             final String dn = dir.getPath();
    212             final String dn2 = dn + "/";
    213             reportSummary(dn,
    214                     new FileFilter() {
    215                         public boolean accept(File file) {
    216                             return file.getPath().startsWith(dn2);
    217                         }
    218             });
    219         }
    220     }
    221 
    222     private void reportSummary(String label, FileFilter filter) {
    223         if (!Config.LOGD) return;
    224 
    225         // log cumulative stats
    226         int total = 0;
    227         int count11 = 0;
    228         int count1N = 0;
    229         int countMN = 0;
    230         int count0N = 0;
    231         int countFail = 0;
    232         int countErrors = 0;
    233 
    234         for (WavFile wf : mWavFiles) {
    235             if (filter == null || filter.accept(wf.mFile)) {
    236                 total++;
    237                 if (wf.mRank == 1 && wf.mTotal == 1) count11++;
    238                 if (wf.mRank == 1 && wf.mTotal >= 1) count1N++;
    239                 if (wf.mRank >= 1 && wf.mTotal >= 1) countMN++;
    240                 if (wf.mRank == 0 && wf.mTotal >= 1) count0N++;
    241                 if (wf.mRank == 0 && wf.mTotal == 0) countFail++;
    242                 if (wf.mRank == -1 && wf.mTotal == -1) countErrors++;
    243             }
    244         }
    245 
    246         String line = String.format(REPORT_FMT,
    247                 countString(count11, total),
    248                 countString(count1N, total),
    249                 countString(countMN, total),
    250                 countString(count0N, total),
    251                 countString(countFail, total),
    252                 countString(countErrors, total),
    253                 "" + total,
    254                 label);
    255         Log.d(TAG, line);
    256     }
    257 
    258     private static String countString(int count, int total) {
    259         return total > 0 ? "" + (100 * count / total) + "%" : "";
    260     }
    261 
    262 }