Home | History | Annotate | Download | only in libtextclassifier 
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #ifndef LIBTEXTCLASSIFIER_TOKEN_FEATURE_EXTRACTOR_H_
     18 #define LIBTEXTCLASSIFIER_TOKEN_FEATURE_EXTRACTOR_H_
     19 
     20 #include <memory>
     21 #include <unordered_set>
     22 #include <vector>
     23 
     24 #include "types.h"
     25 #include "util/strings/stringpiece.h"
     26 #include "util/utf8/unilib.h"
     27 
     28 namespace libtextclassifier2 {
     29 
     30 struct TokenFeatureExtractorOptions {
     31   // Number of buckets used for hashing charactergrams.
     32   int num_buckets = 0;
     33 
     34   // Orders of charactergrams to extract. E.g., 2 means character bigrams, 3
     35   // character trigrams etc.
     36   std::vector<int> chargram_orders;
     37 
     38   // Whether to extract the token case feature.
     39   bool extract_case_feature = false;
     40 
     41   // If true, will use the unicode-aware functionality for extracting features.
     42   bool unicode_aware_features = false;
     43 
     44   // Whether to extract the selection mask feature.
     45   bool extract_selection_mask_feature = false;
     46 
     47   // Regexp features to extract.
     48   std::vector<std::string> regexp_features;
     49 
     50   // Whether to remap digits to a single number.
     51   bool remap_digits = false;
     52 
     53   // Whether to lowercase all tokens.
     54   bool lowercase_tokens = false;
     55 
     56   // Maximum length of a word.
     57   int max_word_length = 20;
     58 
     59   // List of allowed charactergrams. The extracted charactergrams are filtered
     60   // using this list, and charactergrams that are not present are interpreted as
     61   // out-of-vocabulary.
     62   // If no allowed_chargrams are specified, all charactergrams are allowed.
     63   std::unordered_set<std::string> allowed_chargrams;
     64 };
     65 
     66 class TokenFeatureExtractor {
     67  public:
     68   TokenFeatureExtractor(const TokenFeatureExtractorOptions& options,
     69                         const UniLib& unilib);
     70 
     71   // Extracts both the sparse (charactergram) and the dense features from a
     72   // token. is_in_span is a bool indicator whether the token is a part of the
     73   // selection span (true) or not (false).
     74   // The sparse_features output is optional. Fails and returns false if
     75   // dense_fatures in a nullptr.
     76   bool Extract(const Token& token, bool is_in_span,
     77                std::vector<int>* sparse_features,
     78                std::vector<float>* dense_features) const;
     79 
     80   // Extracts the sparse (charactergram) features from the token.
     81   std::vector<int> ExtractCharactergramFeatures(const Token& token) const;
     82 
     83   // Extracts the dense features from the token. is_in_span is a bool indicator
     84   // whether the token is a part of the selection span (true) or not (false).
     85   std::vector<float> ExtractDenseFeatures(const Token& token,
     86                                           bool is_in_span) const;
     87 
     88   int DenseFeaturesCount() const {
     89     int feature_count =
     90         options_.extract_case_feature + options_.extract_selection_mask_feature;
     91     feature_count += regex_patterns_.size();
     92     return feature_count;
     93   }
     94 
     95  protected:
     96   // Hashes given token to given number of buckets.
     97   int HashToken(StringPiece token) const;
     98 
     99   // Extracts the charactergram features from the token in a non-unicode-aware
    100   // way.
    101   std::vector<int> ExtractCharactergramFeaturesAscii(const Token& token) const;
    102 
    103   // Extracts the charactergram features from the token in a unicode-aware way.
    104   std::vector<int> ExtractCharactergramFeaturesUnicode(
    105       const Token& token) const;
    106 
    107  private:
    108   TokenFeatureExtractorOptions options_;
    109   std::vector<std::unique_ptr<UniLib::RegexPattern>> regex_patterns_;
    110   const UniLib& unilib_;
    111 };
    112 
    113 }  // namespace libtextclassifier2
    114 
    115 #endif  // LIBTEXTCLASSIFIER_TOKEN_FEATURE_EXTRACTOR_H_
    116