Home | History | Annotate | Download | only in cpp
      1 #include "opencv2/core/core.hpp"
      2 #include "opencv2/ml/ml.hpp"
      3 
      4 #include <cstdio>
      5 #include <vector>
      6 #include <iostream>
      7 
      8 using namespace std;
      9 using namespace cv;
     10 using namespace cv::ml;
     11 
     12 static void help()
     13 {
     14     printf("\nThe sample demonstrates how to train Random Trees classifier\n"
     15     "(or Boosting classifier, or MLP, or Knearest, or Nbayes, or Support Vector Machines - see main()) using the provided dataset.\n"
     16     "\n"
     17     "We use the sample database letter-recognition.data\n"
     18     "from UCI Repository, here is the link:\n"
     19     "\n"
     20     "Newman, D.J. & Hettich, S. & Blake, C.L. & Merz, C.J. (1998).\n"
     21     "UCI Repository of machine learning databases\n"
     22     "[http://www.ics.uci.edu/~mlearn/MLRepository.html].\n"
     23     "Irvine, CA: University of California, Department of Information and Computer Science.\n"
     24     "\n"
     25     "The dataset consists of 20000 feature vectors along with the\n"
     26     "responses - capital latin letters A..Z.\n"
     27     "The first 16000 (10000 for boosting)) samples are used for training\n"
     28     "and the remaining 4000 (10000 for boosting) - to test the classifier.\n"
     29     "======================================================\n");
     30     printf("\nThis is letter recognition sample.\n"
     31             "The usage: letter_recog [-data <path to letter-recognition.data>] \\\n"
     32             "  [-save <output XML file for the classifier>] \\\n"
     33             "  [-load <XML file with the pre-trained classifier>] \\\n"
     34             "  [-boost|-mlp|-knearest|-nbayes|-svm] # to use boost/mlp/knearest/SVM classifier instead of default Random Trees\n" );
     35 }
     36 
     37 // This function reads data and responses from the file <filename>
     38 static bool
     39 read_num_class_data( const string& filename, int var_count,
     40                      Mat* _data, Mat* _responses )
     41 {
     42     const int M = 1024;
     43     char buf[M+2];
     44 
     45     Mat el_ptr(1, var_count, CV_32F);
     46     int i;
     47     vector<int> responses;
     48 
     49     _data->release();
     50     _responses->release();
     51 
     52     FILE* f = fopen( filename.c_str(), "rt" );
     53     if( !f )
     54     {
     55         cout << "Could not read the database " << filename << endl;
     56         return false;
     57     }
     58 
     59     for(;;)
     60     {
     61         char* ptr;
     62         if( !fgets( buf, M, f ) || !strchr( buf, ',' ) )
     63             break;
     64         responses.push_back((int)buf[0]);
     65         ptr = buf+2;
     66         for( i = 0; i < var_count; i++ )
     67         {
     68             int n = 0;
     69             sscanf( ptr, "%f%n", &el_ptr.at<float>(i), &n );
     70             ptr += n + 1;
     71         }
     72         if( i < var_count )
     73             break;
     74         _data->push_back(el_ptr);
     75     }
     76     fclose(f);
     77     Mat(responses).copyTo(*_responses);
     78 
     79     cout << "The database " << filename << " is loaded.\n";
     80 
     81     return true;
     82 }
     83 
     84 template<typename T>
     85 static Ptr<T> load_classifier(const string& filename_to_load)
     86 {
     87     // load classifier from the specified file
     88     Ptr<T> model = StatModel::load<T>( filename_to_load );
     89     if( model.empty() )
     90         cout << "Could not read the classifier " << filename_to_load << endl;
     91     else
     92         cout << "The classifier " << filename_to_load << " is loaded.\n";
     93 
     94     return model;
     95 }
     96 
     97 static Ptr<TrainData>
     98 prepare_train_data(const Mat& data, const Mat& responses, int ntrain_samples)
     99 {
    100     Mat sample_idx = Mat::zeros( 1, data.rows, CV_8U );
    101     Mat train_samples = sample_idx.colRange(0, ntrain_samples);
    102     train_samples.setTo(Scalar::all(1));
    103 
    104     int nvars = data.cols;
    105     Mat var_type( nvars + 1, 1, CV_8U );
    106     var_type.setTo(Scalar::all(VAR_ORDERED));
    107     var_type.at<uchar>(nvars) = VAR_CATEGORICAL;
    108 
    109     return TrainData::create(data, ROW_SAMPLE, responses,
    110                              noArray(), sample_idx, noArray(), var_type);
    111 }
    112 
    113 inline TermCriteria TC(int iters, double eps)
    114 {
    115     return TermCriteria(TermCriteria::MAX_ITER + (eps > 0 ? TermCriteria::EPS : 0), iters, eps);
    116 }
    117 
    118 static void test_and_save_classifier(const Ptr<StatModel>& model,
    119                                      const Mat& data, const Mat& responses,
    120                                      int ntrain_samples, int rdelta,
    121                                      const string& filename_to_save)
    122 {
    123     int i, nsamples_all = data.rows;
    124     double train_hr = 0, test_hr = 0;
    125 
    126     // compute prediction error on train and test data
    127     for( i = 0; i < nsamples_all; i++ )
    128     {
    129         Mat sample = data.row(i);
    130 
    131         float r = model->predict( sample );
    132         r = std::abs(r + rdelta - responses.at<int>(i)) <= FLT_EPSILON ? 1.f : 0.f;
    133 
    134         if( i < ntrain_samples )
    135             train_hr += r;
    136         else
    137             test_hr += r;
    138     }
    139 
    140     test_hr /= nsamples_all - ntrain_samples;
    141     train_hr = ntrain_samples > 0 ? train_hr/ntrain_samples : 1.;
    142 
    143     printf( "Recognition rate: train = %.1f%%, test = %.1f%%\n",
    144             train_hr*100., test_hr*100. );
    145 
    146     if( !filename_to_save.empty() )
    147     {
    148         model->save( filename_to_save );
    149     }
    150 }
    151 
    152 
    153 static bool
    154 build_rtrees_classifier( const string& data_filename,
    155                          const string& filename_to_save,
    156                          const string& filename_to_load )
    157 {
    158     Mat data;
    159     Mat responses;
    160     bool ok = read_num_class_data( data_filename, 16, &data, &responses );
    161     if( !ok )
    162         return ok;
    163 
    164     Ptr<RTrees> model;
    165 
    166     int nsamples_all = data.rows;
    167     int ntrain_samples = (int)(nsamples_all*0.8);
    168 
    169     // Create or load Random Trees classifier
    170     if( !filename_to_load.empty() )
    171     {
    172         model = load_classifier<RTrees>(filename_to_load);
    173         if( model.empty() )
    174             return false;
    175         ntrain_samples = 0;
    176     }
    177     else
    178     {
    179         // create classifier by using <data> and <responses>
    180         cout << "Training the classifier ...\n";
    181 //        Params( int maxDepth, int minSampleCount,
    182 //                   double regressionAccuracy, bool useSurrogates,
    183 //                   int maxCategories, const Mat& priors,
    184 //                   bool calcVarImportance, int nactiveVars,
    185 //                   TermCriteria termCrit );
    186         Ptr<TrainData> tdata = prepare_train_data(data, responses, ntrain_samples);
    187         model = RTrees::create();
    188         model->setMaxDepth(10);
    189         model->setMinSampleCount(10);
    190         model->setRegressionAccuracy(0);
    191         model->setUseSurrogates(false);
    192         model->setMaxCategories(15);
    193         model->setPriors(Mat());
    194         model->setCalculateVarImportance(true);
    195         model->setActiveVarCount(4);
    196         model->setTermCriteria(TC(100,0.01f));
    197         model->train(tdata);
    198         cout << endl;
    199     }
    200 
    201     test_and_save_classifier(model, data, responses, ntrain_samples, 0, filename_to_save);
    202     cout << "Number of trees: " << model->getRoots().size() << endl;
    203 
    204     // Print variable importance
    205     Mat var_importance = model->getVarImportance();
    206     if( !var_importance.empty() )
    207     {
    208         double rt_imp_sum = sum( var_importance )[0];
    209         printf("var#\timportance (in %%):\n");
    210         int i, n = (int)var_importance.total();
    211         for( i = 0; i < n; i++ )
    212             printf( "%-2d\t%-4.1f\n", i, 100.f*var_importance.at<float>(i)/rt_imp_sum);
    213     }
    214 
    215     return true;
    216 }
    217 
    218 
    219 static bool
    220 build_boost_classifier( const string& data_filename,
    221                         const string& filename_to_save,
    222                         const string& filename_to_load )
    223 {
    224     const int class_count = 26;
    225     Mat data;
    226     Mat responses;
    227     Mat weak_responses;
    228 
    229     bool ok = read_num_class_data( data_filename, 16, &data, &responses );
    230     if( !ok )
    231         return ok;
    232 
    233     int i, j, k;
    234     Ptr<Boost> model;
    235 
    236     int nsamples_all = data.rows;
    237     int ntrain_samples = (int)(nsamples_all*0.5);
    238     int var_count = data.cols;
    239 
    240     // Create or load Boosted Tree classifier
    241     if( !filename_to_load.empty() )
    242     {
    243         model = load_classifier<Boost>(filename_to_load);
    244         if( model.empty() )
    245             return false;
    246         ntrain_samples = 0;
    247     }
    248     else
    249     {
    250         // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    251         //
    252         // As currently boosted tree classifier in MLL can only be trained
    253         // for 2-class problems, we transform the training database by
    254         // "unrolling" each training sample as many times as the number of
    255         // classes (26) that we have.
    256         //
    257         // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    258 
    259         Mat new_data( ntrain_samples*class_count, var_count + 1, CV_32F );
    260         Mat new_responses( ntrain_samples*class_count, 1, CV_32S );
    261 
    262         // 1. unroll the database type mask
    263         printf( "Unrolling the database...\n");
    264         for( i = 0; i < ntrain_samples; i++ )
    265         {
    266             const float* data_row = data.ptr<float>(i);
    267             for( j = 0; j < class_count; j++ )
    268             {
    269                 float* new_data_row = (float*)new_data.ptr<float>(i*class_count+j);
    270                 memcpy(new_data_row, data_row, var_count*sizeof(data_row[0]));
    271                 new_data_row[var_count] = (float)j;
    272                 new_responses.at<int>(i*class_count + j) = responses.at<int>(i) == j+'A';
    273             }
    274         }
    275 
    276         Mat var_type( 1, var_count + 2, CV_8U );
    277         var_type.setTo(Scalar::all(VAR_ORDERED));
    278         var_type.at<uchar>(var_count) = var_type.at<uchar>(var_count+1) = VAR_CATEGORICAL;
    279 
    280         Ptr<TrainData> tdata = TrainData::create(new_data, ROW_SAMPLE, new_responses,
    281                                                  noArray(), noArray(), noArray(), var_type);
    282         vector<double> priors(2);
    283         priors[0] = 1;
    284         priors[1] = 26;
    285 
    286         cout << "Training the classifier (may take a few minutes)...\n";
    287         model = Boost::create();
    288         model->setBoostType(Boost::GENTLE);
    289         model->setWeakCount(100);
    290         model->setWeightTrimRate(0.95);
    291         model->setMaxDepth(5);
    292         model->setUseSurrogates(false);
    293         model->setPriors(Mat(priors));
    294         model->train(tdata);
    295         cout << endl;
    296     }
    297 
    298     Mat temp_sample( 1, var_count + 1, CV_32F );
    299     float* tptr = temp_sample.ptr<float>();
    300 
    301     // compute prediction error on train and test data
    302     double train_hr = 0, test_hr = 0;
    303     for( i = 0; i < nsamples_all; i++ )
    304     {
    305         int best_class = 0;
    306         double max_sum = -DBL_MAX;
    307         const float* ptr = data.ptr<float>(i);
    308         for( k = 0; k < var_count; k++ )
    309             tptr[k] = ptr[k];
    310 
    311         for( j = 0; j < class_count; j++ )
    312         {
    313             tptr[var_count] = (float)j;
    314             float s = model->predict( temp_sample, noArray(), StatModel::RAW_OUTPUT );
    315             if( max_sum < s )
    316             {
    317                 max_sum = s;
    318                 best_class = j + 'A';
    319             }
    320         }
    321 
    322         double r = std::abs(best_class - responses.at<int>(i)) < FLT_EPSILON ? 1 : 0;
    323         if( i < ntrain_samples )
    324             train_hr += r;
    325         else
    326             test_hr += r;
    327     }
    328 
    329     test_hr /= nsamples_all-ntrain_samples;
    330     train_hr = ntrain_samples > 0 ? train_hr/ntrain_samples : 1.;
    331     printf( "Recognition rate: train = %.1f%%, test = %.1f%%\n",
    332             train_hr*100., test_hr*100. );
    333 
    334     cout << "Number of trees: " << model->getRoots().size() << endl;
    335 
    336     // Save classifier to file if needed
    337     if( !filename_to_save.empty() )
    338         model->save( filename_to_save );
    339 
    340     return true;
    341 }
    342 
    343 
    344 static bool
    345 build_mlp_classifier( const string& data_filename,
    346                       const string& filename_to_save,
    347                       const string& filename_to_load )
    348 {
    349     const int class_count = 26;
    350     Mat data;
    351     Mat responses;
    352 
    353     bool ok = read_num_class_data( data_filename, 16, &data, &responses );
    354     if( !ok )
    355         return ok;
    356 
    357     Ptr<ANN_MLP> model;
    358 
    359     int nsamples_all = data.rows;
    360     int ntrain_samples = (int)(nsamples_all*0.8);
    361 
    362     // Create or load MLP classifier
    363     if( !filename_to_load.empty() )
    364     {
    365         model = load_classifier<ANN_MLP>(filename_to_load);
    366         if( model.empty() )
    367             return false;
    368         ntrain_samples = 0;
    369     }
    370     else
    371     {
    372         // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    373         //
    374         // MLP does not support categorical variables by explicitly.
    375         // So, instead of the output class label, we will use
    376         // a binary vector of <class_count> components for training and,
    377         // therefore, MLP will give us a vector of "probabilities" at the
    378         // prediction stage
    379         //
    380         // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    381 
    382         Mat train_data = data.rowRange(0, ntrain_samples);
    383         Mat train_responses = Mat::zeros( ntrain_samples, class_count, CV_32F );
    384 
    385         // 1. unroll the responses
    386         cout << "Unrolling the responses...\n";
    387         for( int i = 0; i < ntrain_samples; i++ )
    388         {
    389             int cls_label = responses.at<int>(i) - 'A';
    390             train_responses.at<float>(i, cls_label) = 1.f;
    391         }
    392 
    393         // 2. train classifier
    394         int layer_sz[] = { data.cols, 100, 100, class_count };
    395         int nlayers = (int)(sizeof(layer_sz)/sizeof(layer_sz[0]));
    396         Mat layer_sizes( 1, nlayers, CV_32S, layer_sz );
    397 
    398 #if 1
    399         int method = ANN_MLP::BACKPROP;
    400         double method_param = 0.001;
    401         int max_iter = 300;
    402 #else
    403         int method = ANN_MLP::RPROP;
    404         double method_param = 0.1;
    405         int max_iter = 1000;
    406 #endif
    407 
    408         Ptr<TrainData> tdata = TrainData::create(train_data, ROW_SAMPLE, train_responses);
    409 
    410         cout << "Training the classifier (may take a few minutes)...\n";
    411         model = ANN_MLP::create();
    412         model->setLayerSizes(layer_sizes);
    413         model->setActivationFunction(ANN_MLP::SIGMOID_SYM, 0, 0);
    414         model->setTermCriteria(TC(max_iter,0));
    415         model->setTrainMethod(method, method_param);
    416         model->train(tdata);
    417         cout << endl;
    418     }
    419 
    420     test_and_save_classifier(model, data, responses, ntrain_samples, 'A', filename_to_save);
    421     return true;
    422 }
    423 
    424 static bool
    425 build_knearest_classifier( const string& data_filename, int K )
    426 {
    427     Mat data;
    428     Mat responses;
    429     bool ok = read_num_class_data( data_filename, 16, &data, &responses );
    430     if( !ok )
    431         return ok;
    432 
    433 
    434     int nsamples_all = data.rows;
    435     int ntrain_samples = (int)(nsamples_all*0.8);
    436 
    437     // create classifier by using <data> and <responses>
    438     cout << "Training the classifier ...\n";
    439     Ptr<TrainData> tdata = prepare_train_data(data, responses, ntrain_samples);
    440     Ptr<KNearest> model = KNearest::create();
    441     model->setDefaultK(K);
    442     model->setIsClassifier(true);
    443     model->train(tdata);
    444     cout << endl;
    445 
    446     test_and_save_classifier(model, data, responses, ntrain_samples, 0, string());
    447     return true;
    448 }
    449 
    450 static bool
    451 build_nbayes_classifier( const string& data_filename )
    452 {
    453     Mat data;
    454     Mat responses;
    455     bool ok = read_num_class_data( data_filename, 16, &data, &responses );
    456     if( !ok )
    457         return ok;
    458 
    459     Ptr<NormalBayesClassifier> model;
    460 
    461     int nsamples_all = data.rows;
    462     int ntrain_samples = (int)(nsamples_all*0.8);
    463 
    464     // create classifier by using <data> and <responses>
    465     cout << "Training the classifier ...\n";
    466     Ptr<TrainData> tdata = prepare_train_data(data, responses, ntrain_samples);
    467     model = NormalBayesClassifier::create();
    468     model->train(tdata);
    469     cout << endl;
    470 
    471     test_and_save_classifier(model, data, responses, ntrain_samples, 0, string());
    472     return true;
    473 }
    474 
    475 static bool
    476 build_svm_classifier( const string& data_filename,
    477                       const string& filename_to_save,
    478                       const string& filename_to_load )
    479 {
    480     Mat data;
    481     Mat responses;
    482     bool ok = read_num_class_data( data_filename, 16, &data, &responses );
    483     if( !ok )
    484         return ok;
    485 
    486     Ptr<SVM> model;
    487 
    488     int nsamples_all = data.rows;
    489     int ntrain_samples = (int)(nsamples_all*0.8);
    490 
    491     // Create or load Random Trees classifier
    492     if( !filename_to_load.empty() )
    493     {
    494         model = load_classifier<SVM>(filename_to_load);
    495         if( model.empty() )
    496             return false;
    497         ntrain_samples = 0;
    498     }
    499     else
    500     {
    501         // create classifier by using <data> and <responses>
    502         cout << "Training the classifier ...\n";
    503         Ptr<TrainData> tdata = prepare_train_data(data, responses, ntrain_samples);
    504         model = SVM::create();
    505         model->setType(SVM::C_SVC);
    506         model->setKernel(SVM::LINEAR);
    507         model->setC(1);
    508         model->train(tdata);
    509         cout << endl;
    510     }
    511 
    512     test_and_save_classifier(model, data, responses, ntrain_samples, 0, filename_to_save);
    513     return true;
    514 }
    515 
    516 int main( int argc, char *argv[] )
    517 {
    518     string filename_to_save = "";
    519     string filename_to_load = "";
    520     string data_filename = "../data/letter-recognition.data";
    521     int method = 0;
    522 
    523     int i;
    524     for( i = 1; i < argc; i++ )
    525     {
    526         if( strcmp(argv[i],"-data") == 0 ) // flag "-data letter_recognition.xml"
    527         {
    528             i++;
    529             data_filename = argv[i];
    530         }
    531         else if( strcmp(argv[i],"-save") == 0 ) // flag "-save filename.xml"
    532         {
    533             i++;
    534             filename_to_save = argv[i];
    535         }
    536         else if( strcmp(argv[i],"-load") == 0) // flag "-load filename.xml"
    537         {
    538             i++;
    539             filename_to_load = argv[i];
    540         }
    541         else if( strcmp(argv[i],"-boost") == 0)
    542         {
    543             method = 1;
    544         }
    545         else if( strcmp(argv[i],"-mlp") == 0 )
    546         {
    547             method = 2;
    548         }
    549         else if( strcmp(argv[i], "-knearest") == 0 || strcmp(argv[i], "-knn") == 0 )
    550         {
    551             method = 3;
    552         }
    553         else if( strcmp(argv[i], "-nbayes") == 0)
    554         {
    555             method = 4;
    556         }
    557         else if( strcmp(argv[i], "-svm") == 0)
    558         {
    559             method = 5;
    560         }
    561         else
    562             break;
    563     }
    564 
    565     if( i < argc ||
    566         (method == 0 ?
    567         build_rtrees_classifier( data_filename, filename_to_save, filename_to_load ) :
    568         method == 1 ?
    569         build_boost_classifier( data_filename, filename_to_save, filename_to_load ) :
    570         method == 2 ?
    571         build_mlp_classifier( data_filename, filename_to_save, filename_to_load ) :
    572         method == 3 ?
    573         build_knearest_classifier( data_filename, 10 ) :
    574         method == 4 ?
    575         build_nbayes_classifier( data_filename) :
    576         method == 5 ?
    577         build_svm_classifier( data_filename, filename_to_save, filename_to_load ):
    578         -1) < 0)
    579     {
    580         help();
    581     }
    582     return 0;
    583 }
    584