1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "smartselect/token-feature-extractor.h" 18 19 #include <string> 20 21 #include "util/base/logging.h" 22 #include "util/hash/farmhash.h" 23 #include "util/strings/stringpiece.h" 24 #include "util/utf8/unicodetext.h" 25 #include "unicode/regex.h" 26 #include "unicode/uchar.h" 27 28 namespace libtextclassifier { 29 30 namespace { 31 32 std::string RemapTokenAscii(const std::string& token, 33 const TokenFeatureExtractorOptions& options) { 34 if (!options.remap_digits && !options.lowercase_tokens) { 35 return token; 36 } 37 38 std::string copy = token; 39 for (int i = 0; i < token.size(); ++i) { 40 if (options.remap_digits && isdigit(copy[i])) { 41 copy[i] = '0'; 42 } 43 if (options.lowercase_tokens) { 44 copy[i] = tolower(copy[i]); 45 } 46 } 47 return copy; 48 } 49 50 void RemapTokenUnicode(const std::string& token, 51 const TokenFeatureExtractorOptions& options, 52 UnicodeText* remapped) { 53 if (!options.remap_digits && !options.lowercase_tokens) { 54 // Leave remapped untouched. 55 return; 56 } 57 58 UnicodeText word = UTF8ToUnicodeText(token, /*do_copy=*/false); 59 icu::UnicodeString icu_string; 60 for (auto it = word.begin(); it != word.end(); ++it) { 61 if (options.remap_digits && u_isdigit(*it)) { 62 icu_string.append('0'); 63 } else if (options.lowercase_tokens) { 64 icu_string.append(u_tolower(*it)); 65 } else { 66 icu_string.append(*it); 67 } 68 } 69 std::string utf8_str; 70 icu_string.toUTF8String(utf8_str); 71 remapped->CopyUTF8(utf8_str.data(), utf8_str.length()); 72 } 73 74 } // namespace 75 76 TokenFeatureExtractor::TokenFeatureExtractor( 77 const TokenFeatureExtractorOptions& options) 78 : options_(options) { 79 UErrorCode status; 80 for (const std::string& pattern : options.regexp_features) { 81 status = U_ZERO_ERROR; 82 regex_patterns_.push_back( 83 std::unique_ptr<icu::RegexPattern>(icu::RegexPattern::compile( 84 icu::UnicodeString(pattern.c_str(), pattern.size(), "utf-8"), 0, 85 status))); 86 if (U_FAILURE(status)) { 87 TC_LOG(WARNING) << "Failed to load pattern" << pattern; 88 } 89 } 90 } 91 92 int TokenFeatureExtractor::HashToken(StringPiece token) const { 93 return tcfarmhash::Fingerprint64(token) % options_.num_buckets; 94 } 95 96 std::vector<int> TokenFeatureExtractor::ExtractCharactergramFeatures( 97 const Token& token) const { 98 if (options_.unicode_aware_features) { 99 return ExtractCharactergramFeaturesUnicode(token); 100 } else { 101 return ExtractCharactergramFeaturesAscii(token); 102 } 103 } 104 105 std::vector<int> TokenFeatureExtractor::ExtractCharactergramFeaturesAscii( 106 const Token& token) const { 107 std::vector<int> result; 108 if (token.is_padding || token.value.empty()) { 109 result.push_back(HashToken("<PAD>")); 110 } else { 111 const std::string word = RemapTokenAscii(token.value, options_); 112 113 // Trim words that are over max_word_length characters. 114 const int max_word_length = options_.max_word_length; 115 std::string feature_word; 116 if (word.size() > max_word_length) { 117 feature_word = 118 "^" + word.substr(0, max_word_length / 2) + "\1" + 119 word.substr(word.size() - max_word_length / 2, max_word_length / 2) + 120 "$"; 121 } else { 122 // Add a prefix and suffix to the word. 123 feature_word = "^" + word + "$"; 124 } 125 126 // Upper-bound the number of charactergram extracted to avoid resizing. 127 result.reserve(options_.chargram_orders.size() * feature_word.size()); 128 129 // Generate the character-grams. 130 for (int chargram_order : options_.chargram_orders) { 131 if (chargram_order == 1) { 132 for (int i = 1; i < feature_word.size() - 1; ++i) { 133 result.push_back( 134 HashToken(StringPiece(feature_word, /*offset=*/i, /*len=*/1))); 135 } 136 } else { 137 for (int i = 0; 138 i < static_cast<int>(feature_word.size()) - chargram_order + 1; 139 ++i) { 140 result.push_back(HashToken( 141 StringPiece(feature_word, /*offset=*/i, /*len=*/chargram_order))); 142 } 143 } 144 } 145 } 146 return result; 147 } 148 149 std::vector<int> TokenFeatureExtractor::ExtractCharactergramFeaturesUnicode( 150 const Token& token) const { 151 std::vector<int> result; 152 if (token.is_padding || token.value.empty()) { 153 result.push_back(HashToken("<PAD>")); 154 } else { 155 UnicodeText word = UTF8ToUnicodeText(token.value, /*do_copy=*/false); 156 RemapTokenUnicode(token.value, options_, &word); 157 158 // Trim the word if needed by finding a left-cut point and right-cut point. 159 auto left_cut = word.begin(); 160 auto right_cut = word.end(); 161 for (int i = 0; i < options_.max_word_length / 2; i++) { 162 if (left_cut < right_cut) { 163 ++left_cut; 164 } 165 if (left_cut < right_cut) { 166 --right_cut; 167 } 168 } 169 170 std::string feature_word; 171 if (left_cut == right_cut) { 172 feature_word = "^" + word.UTF8Substring(word.begin(), word.end()) + "$"; 173 } else { 174 // clang-format off 175 feature_word = "^" + 176 word.UTF8Substring(word.begin(), left_cut) + 177 "\1" + 178 word.UTF8Substring(right_cut, word.end()) + 179 "$"; 180 // clang-format on 181 } 182 183 const UnicodeText feature_word_unicode = 184 UTF8ToUnicodeText(feature_word, /*do_copy=*/false); 185 186 // Upper-bound the number of charactergram extracted to avoid resizing. 187 result.reserve(options_.chargram_orders.size() * feature_word.size()); 188 189 // Generate the character-grams. 190 for (int chargram_order : options_.chargram_orders) { 191 UnicodeText::const_iterator it_start = feature_word_unicode.begin(); 192 UnicodeText::const_iterator it_end = feature_word_unicode.end(); 193 if (chargram_order == 1) { 194 ++it_start; 195 --it_end; 196 } 197 198 UnicodeText::const_iterator it_chargram_start = it_start; 199 UnicodeText::const_iterator it_chargram_end = it_start; 200 bool chargram_is_complete = true; 201 for (int i = 0; i < chargram_order; ++i) { 202 if (it_chargram_end == it_end) { 203 chargram_is_complete = false; 204 break; 205 } 206 ++it_chargram_end; 207 } 208 if (!chargram_is_complete) { 209 continue; 210 } 211 212 for (; it_chargram_end <= it_end; 213 ++it_chargram_start, ++it_chargram_end) { 214 const int length_bytes = 215 it_chargram_end.utf8_data() - it_chargram_start.utf8_data(); 216 result.push_back(HashToken( 217 StringPiece(it_chargram_start.utf8_data(), length_bytes))); 218 } 219 } 220 } 221 return result; 222 } 223 224 bool TokenFeatureExtractor::Extract(const Token& token, bool is_in_span, 225 std::vector<int>* sparse_features, 226 std::vector<float>* dense_features) const { 227 if (sparse_features == nullptr || dense_features == nullptr) { 228 return false; 229 } 230 231 *sparse_features = ExtractCharactergramFeatures(token); 232 233 if (options_.extract_case_feature) { 234 if (options_.unicode_aware_features) { 235 UnicodeText token_unicode = 236 UTF8ToUnicodeText(token.value, /*do_copy=*/false); 237 if (!token.value.empty() && u_isupper(*token_unicode.begin())) { 238 dense_features->push_back(1.0); 239 } else { 240 dense_features->push_back(-1.0); 241 } 242 } else { 243 if (!token.value.empty() && isupper(*token.value.begin())) { 244 dense_features->push_back(1.0); 245 } else { 246 dense_features->push_back(-1.0); 247 } 248 } 249 } 250 251 if (options_.extract_selection_mask_feature) { 252 if (is_in_span) { 253 dense_features->push_back(1.0); 254 } else { 255 if (options_.unicode_aware_features) { 256 dense_features->push_back(-1.0); 257 } else { 258 dense_features->push_back(0.0); 259 } 260 } 261 } 262 263 // Add regexp features. 264 if (!regex_patterns_.empty()) { 265 icu::UnicodeString unicode_str(token.value.c_str(), token.value.size(), 266 "utf-8"); 267 for (int i = 0; i < regex_patterns_.size(); ++i) { 268 if (!regex_patterns_[i].get()) { 269 dense_features->push_back(-1.0); 270 continue; 271 } 272 273 // Check for match. 274 UErrorCode status = U_ZERO_ERROR; 275 std::unique_ptr<icu::RegexMatcher> matcher( 276 regex_patterns_[i]->matcher(unicode_str, status)); 277 if (matcher->find()) { 278 dense_features->push_back(1.0); 279 } else { 280 dense_features->push_back(-1.0); 281 } 282 } 283 } 284 return true; 285 } 286 287 } // namespace libtextclassifier 288