Home | History | Annotate | Download | only in utils 
      1 /*
      2  * Copyright (C) 2013, The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #ifndef LATINIME_PROBABILITY_UTILS_H
     18 #define LATINIME_PROBABILITY_UTILS_H
     19 
     20 #include <algorithm>
     21 #include <cmath>
     22 
     23 #include "defines.h"
     24 
     25 namespace latinime {
     26 
     27 // TODO: Quit using bigram probability to indicate the delta.
     28 class ProbabilityUtils {
     29  public:
     30     static AK_FORCE_INLINE int backoff(const int unigramProbability) {
     31         return unigramProbability;
     32         // For some reason, applying the backoff weight gives bad results in tests. To apply the
     33         // backoff weight, we divide the probability by 2, which in our storing format means
     34         // decreasing the score by 8.
     35         // TODO: figure out what's wrong with this.
     36         // return unigramProbability > 8 ?
     37         //         unigramProbability - 8 : (0 == unigramProbability ? 0 : 8);
     38     }
     39 
     40     static AK_FORCE_INLINE int computeProbabilityForBigram(
     41             const int unigramProbability, const int bigramProbability) {
     42         // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want
     43         // the unigram probability to be the median value of the 17th step from the top. A value of
     44         // 0 for the bigram probability represents the middle of the 16th step from the top,
     45         // while a value of 15 represents the middle of the top step.
     46         // See makedict.BinaryDictEncoder#makeBigramFlags for details.
     47         const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
     48                 / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
     49         return unigramProbability
     50                 + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize);
     51     }
     52 
     53     // Encode probability using the same way as we are doing for main dictionaries.
     54     static AK_FORCE_INLINE int encodeRawProbability(const float rawProbability) {
     55         const float probability = static_cast<float>(MAX_PROBABILITY)
     56                 + log2f(rawProbability) * PROBABILITY_ENCODING_SCALER;
     57         if (probability < 0.0f) {
     58             return 0;
     59         }
     60         return std::min(static_cast<int>(probability + 0.5f), MAX_PROBABILITY);
     61     }
     62 
     63  private:
     64     DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils);
     65 
     66     static const float PROBABILITY_ENCODING_SCALER;
     67 };
     68 }
     69 #endif /* LATINIME_PROBABILITY_UTILS_H */
     70