1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_IMPL_H_ 6 #define ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_IMPL_H_ 7 8 #include "encodings/lang_enc.h" 9 #include "encodings/compact_lang_det/win/cld_basictypes.h" 10 11 12 static const int kCLDFlagFinish = 1; 13 static const int kCLDFlagSqueeze = 2; 14 static const int kCLDFlagRepeats = 4; 15 static const int kCLDFlagTop40 = 8; 16 static const int kCLDFlagShort = 16; 17 static const int kCLDFlagHint = 32; // Experimental, undebugged 18 static const int kCLDFlagUseWords = 64; 19 20 /*** 21 22 Flag meanings: 23 24 Flags are used in the context of a recursive call from Detect to itself, 25 trying to deal in a more restrictive way with input that was not reliably 26 identified in the top-level call. 27 28 Finish -- Do not further recurse; return whatever result ensues, even if it is 29 unreliable. Typically set in any recursive call to take a second try 30 on unreliable text. 31 32 Squeeze -- For each text run, do an inplace cheapsqueeze to remove chunks of 33 highly repetitive text and chunks of text with too many 1- and 34 2-letter words. This avoids scoring repetitive or useless non-text 35 crap in large files such bogus JPEGs within an HTML file. 36 37 Repeats -- When scoring a text run, do a cheap prediction of each character 38 and do not score a unigram/quadgram if the last character of same is 39 correctly predicted. This is a slower, finer-grained form of 40 cheapsqueeze, typically used when the first pass got unreliable 41 results. 42 43 Top40 -- Restrict the set of scored languages to the Google "Top 40*", which is 44 actually 38 languages. This gets rid of about 110 language that 45 represent about 0.7% of the web. Typically used when the first pass 46 got unreliable results. 47 48 Short -- Use trigram (three letter) scoring instad of quadgrams. Restricted to 49 the top 40* languages, Latin and Cyrillic scripts only. 50 Not as precise as quadgrams, but it gives some plausible result on 51 1- or 2-word text in major languages. 52 53 Hint -- EXPERIMENTAL flag for compact_lang_det_test.cc to indicate a language 54 hint supplied in parameter plus_one. 55 56 UseWords -- In additon to scoring quad/uni/nil-grams, score complete words 57 58 59 Tentative decision logic: 60 61 In the middle of first pass -- After 4KB of text, look at the front 256 bytes 62 of every full 4KB buffer. If it compresses very well (say 3:1) or has 63 lots of spaces (say 1 of every 4 bytes), assume that the input is 64 large and contains lots of bogus non-text. Recurse, passing the 65 Squeeze flag to strip out chunks of this non-text. 66 67 At the end of the first pass -- 68 If the top language is reliable and >= 70% of the document, return. 69 Else if the top language is reliable and top+2nd >= say 94%, return. 70 Else, either the top language is not reliable or there is a lot of 71 other crap. 72 ***/ 73 74 75 namespace CompactLangDet { 76 struct DetectionTables; 77 } // namespace CompactLangDet 78 79 80 namespace CompactLangDetImpl { 81 // Scan interchange-valid UTF-8 bytes and detect most likely language, 82 // or set of languages. 83 // 84 // Design goals: 85 // Skip over big stretches of HTML tags 86 // Able to return ranges of different languages 87 // Relatively small tables and relatively fast processing 88 // Thread safe 89 // 90 91 typedef struct { 92 int perscript_count; 93 const Language* perscript_lang; 94 } PerScriptPair; 95 96 typedef struct { 97 // Constants for hashing 4-7 byte quadgram to 32 bits 98 const int kQuadHashB4Shift; 99 const int kQuadHashB4bShift; 100 const int kQuadHashB5Shift; 101 const int kQuadHashB5bShift; 102 // Constants for hashing 32 bits to kQuadKeyTable subscript/key 103 const int kHashvalToSubShift; 104 const uint32 kHashvalToSubMask; 105 const int kHashvalToKeyShift; 106 const uint32 kHashvalToKeyMask; 107 const int kHashvalAssociativity; 108 // Pointers to the actual tables 109 const PerScriptPair* kPerScriptPair; 110 const uint16* kQuadKeyTable; 111 const uint32* kQuadValueTable; 112 } LangDetObj; 113 114 // For HTML documents, tags are skipped, along with <script> ... </script> 115 // and <style> ... </style> sequences, and entities are expanded. 116 // 117 // We distinguish between bytes of the raw input buffer and bytes of non-tag 118 // text letters. Since tags can be over 50% of the bytes of an HTML Page, 119 // and are nearly all seven-bit ASCII English, we prefer to distinguish 120 // language mixture fractions based on just the non-tag text. 121 // 122 // Inputs: text and text_length 123 // is_plain_text if true says to NOT parse/skip HTML tags nor entities 124 // Outputs: 125 // language3 is an array of the top 3 languages or UNKNOWN_LANGUAGE 126 // percent3 is an array of the text percentages 0..100 of the top 3 languages 127 // normalized_score3 is an array of internal scores, normalized to the 128 // average score for each language over a body of training text. A 129 // normalized score significantly away from 1.0 indicates very skewed text 130 // or gibberish. 131 // 132 // text_bytes is the amount of non-tag/letters-only text found 133 // is_reliable set true if the returned Language is at least 2**30 times more 134 // probable then the second-best Language 135 // 136 // Return value: the most likely Language for the majority of the input text 137 // Length 0 input and text with no reliable letter sequences returns 138 // UNKNOWN_LANGUAGE 139 // 140 // Subsetting: For fast detection over large documents, these routines will 141 // scan non-tag text of the initial part of a document, then will 142 // skip 4-16 bytes and subsample text in the rest of the document, up to a 143 // fixed limit (currently 160KB of non-tag letters). 144 // 145 146 Language DetectLanguageSummaryV25( 147 const CompactLangDet::DetectionTables* tables, 148 const char* buffer, 149 int buffer_length, 150 bool is_plain_text, 151 const char* tld_hint, // "id" boosts Indonesian 152 int encoding_hint, // SJS boosts Japanese 153 Language language_hint, // ITALIAN boosts it 154 bool allow_extended_lang, 155 int flags, 156 Language plus_one, 157 Language* language3, 158 int* percent3, 159 double* normalized_score3, 160 int* text_bytes, 161 bool* is_reliable); 162 163 // For unit testing: 164 // Remove portions of text that have a high density of spaces, or that are 165 // overly repetitive, squeezing the remaining text in-place to the front 166 // of the input buffer. 167 // Return the new, possibly-shorter length 168 int CheapSqueezeInplace(char* isrc, int srclen, int ichunksize); 169 }; // End namespace CompactLangDetImpl 170 171 #endif // ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_IMPL_H_ 172
