1 /* 2 * Copyright (C) 2013, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef LATINIME_PROBABILITY_UTILS_H 18 #define LATINIME_PROBABILITY_UTILS_H 19 20 #include "defines.h" 21 22 namespace latinime { 23 24 // TODO: Quit using bigram probability to indicate the delta. 25 class ProbabilityUtils { 26 public: 27 static AK_FORCE_INLINE int backoff(const int unigramProbability) { 28 return unigramProbability; 29 // For some reason, applying the backoff weight gives bad results in tests. To apply the 30 // backoff weight, we divide the probability by 2, which in our storing format means 31 // decreasing the score by 8. 32 // TODO: figure out what's wrong with this. 33 // return unigramProbability > 8 ? 34 // unigramProbability - 8 : (0 == unigramProbability ? 0 : 8); 35 } 36 37 static AK_FORCE_INLINE int computeProbabilityForBigram( 38 const int unigramProbability, const int bigramProbability) { 39 // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want 40 // the unigram probability to be the median value of the 17th step from the top. A value of 41 // 0 for the bigram probability represents the middle of the 16th step from the top, 42 // while a value of 15 represents the middle of the top step. 43 // See makedict.BinaryDictEncoder#makeBigramFlags for details. 44 const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability) 45 / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY); 46 return unigramProbability 47 + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize); 48 } 49 50 private: 51 DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils); 52 }; 53 } 54 #endif /* LATINIME_PROBABILITY_UTILS_H */ 55
