1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 /* 18 * !!!!! DO NOT EDIT THIS FILE !!!!! 19 * 20 * This file was generated from 21 * dictionary/structure/v4/content/bigram_dict_content.cpp 22 */ 23 24 #include "dictionary/structure/backward/v402/content/bigram_dict_content.h" 25 26 #include "dictionary/utils/buffer_with_extendable_buffer.h" 27 28 namespace latinime { 29 namespace backward { 30 namespace v402 { 31 32 const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition( 33 int *const bigramEntryPos) const { 34 const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer(); 35 const int bigramEntryTailPos = (*bigramEntryPos) + getBigramEntrySize(); 36 if (*bigramEntryPos < 0 || bigramEntryTailPos > bigramListBuffer->getTailPosition()) { 37 AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bigramEntryTailPos: %d, " 38 "bufSize: %d", *bigramEntryPos, bigramEntryTailPos, 39 bigramListBuffer->getTailPosition()); 40 ASSERT(false); 41 return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY, 42 Ver4DictConstants::NOT_A_TERMINAL_ID); 43 } 44 const int bigramFlags = bigramListBuffer->readUintAndAdvancePosition( 45 Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, bigramEntryPos); 46 const bool hasNext = (bigramFlags & Ver4DictConstants::BIGRAM_HAS_NEXT_MASK) != 0; 47 int probability = NOT_A_PROBABILITY; 48 int timestamp = NOT_A_TIMESTAMP; 49 int level = 0; 50 int count = 0; 51 if (mHasHistoricalInfo) { 52 timestamp = bigramListBuffer->readUintAndAdvancePosition( 53 Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos); 54 level = bigramListBuffer->readUintAndAdvancePosition( 55 Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, bigramEntryPos); 56 count = bigramListBuffer->readUintAndAdvancePosition( 57 Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos); 58 } else { 59 probability = bigramListBuffer->readUintAndAdvancePosition( 60 Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos); 61 } 62 const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition( 63 Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos); 64 const int targetTerminalId = 65 (encodedTargetTerminalId == Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID) ? 66 Ver4DictConstants::NOT_A_TERMINAL_ID : encodedTargetTerminalId; 67 if (mHasHistoricalInfo) { 68 // Hack for better migration. 69 count += level; 70 const HistoricalInfo historicalInfo(timestamp, level, count); 71 return BigramEntry(hasNext, probability, &historicalInfo, targetTerminalId); 72 } else { 73 return BigramEntry(hasNext, probability, targetTerminalId); 74 } 75 } 76 77 bool BigramDictContent::writeBigramEntryAndAdvancePosition( 78 const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) { 79 BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer(); 80 const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext()); 81 if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags, 82 Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) { 83 AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags); 84 return false; 85 } 86 if (mHasHistoricalInfo) { 87 const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo(); 88 if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimestamp(), 89 Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) { 90 AKLOGE("Cannot write bigram timestamps. pos: %d, timestamp: %d", *entryWritingPos, 91 historicalInfo->getTimestamp()); 92 return false; 93 } 94 if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getLevel(), 95 Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, entryWritingPos)) { 96 AKLOGE("Cannot write bigram level. pos: %d, level: %d", *entryWritingPos, 97 historicalInfo->getLevel()); 98 return false; 99 } 100 if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getCount(), 101 Ver4DictConstants::WORD_COUNT_FIELD_SIZE, entryWritingPos)) { 102 AKLOGE("Cannot write bigram count. pos: %d, count: %d", *entryWritingPos, 103 historicalInfo->getCount()); 104 return false; 105 } 106 } else { 107 if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(), 108 Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) { 109 AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos, 110 bigramEntryToWrite->getProbability()); 111 return false; 112 } 113 } 114 const int targetTerminalIdToWrite = 115 (bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ? 116 Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID : 117 bigramEntryToWrite->getTargetTerminalId(); 118 if (!bigramListBuffer->writeUintAndAdvancePosition(targetTerminalIdToWrite, 119 Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, entryWritingPos)) { 120 AKLOGE("Cannot write bigram target terminal id. pos: %d, target terminal id: %d", 121 *entryWritingPos, bigramEntryToWrite->getTargetTerminalId()); 122 return false; 123 } 124 return true; 125 } 126 127 bool BigramDictContent::copyBigramList(const int bigramListPos, const int toPos, 128 int *const outTailEntryPos) { 129 int readingPos = bigramListPos; 130 int writingPos = toPos; 131 bool hasNext = true; 132 while (hasNext) { 133 const BigramEntry bigramEntry = getBigramEntryAndAdvancePosition(&readingPos); 134 hasNext = bigramEntry.hasNext(); 135 if (!hasNext) { 136 *outTailEntryPos = writingPos; 137 } 138 if (!writeBigramEntryAndAdvancePosition(&bigramEntry, &writingPos)) { 139 AKLOGE("Cannot write bigram entry to copy. pos: %d", writingPos); 140 return false; 141 } 142 } 143 return true; 144 } 145 146 bool BigramDictContent::runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, 147 const BigramDictContent *const originalBigramDictContent, 148 int *const outBigramEntryCount) { 149 for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin(); 150 it != terminalIdMap->end(); ++it) { 151 const int originalBigramListPos = 152 originalBigramDictContent->getBigramListHeadPos(it->first); 153 if (originalBigramListPos == NOT_A_DICT_POS) { 154 // This terminal does not have a bigram list. 155 continue; 156 } 157 const int bigramListPos = getContentBuffer()->getTailPosition(); 158 int bigramEntryCount = 0; 159 // Copy bigram list with GC from original content. 160 if (!runGCBigramList(originalBigramListPos, originalBigramDictContent, bigramListPos, 161 terminalIdMap, &bigramEntryCount)) { 162 AKLOGE("Cannot complete GC for the bigram list. original pos: %d, pos: %d", 163 originalBigramListPos, bigramListPos); 164 return false; 165 } 166 if (bigramEntryCount == 0) { 167 // All bigram entries are useless. This terminal does not have a bigram list. 168 continue; 169 } 170 *outBigramEntryCount += bigramEntryCount; 171 // Set bigram list position to the lookup table. 172 if (!getUpdatableAddressLookupTable()->set(it->second, bigramListPos)) { 173 AKLOGE("Cannot set bigram list position. terminal id: %d, pos: %d", 174 it->second, bigramListPos); 175 return false; 176 } 177 } 178 return true; 179 } 180 181 // Returns whether GC for the bigram list was succeeded or not. 182 bool BigramDictContent::runGCBigramList(const int bigramListPos, 183 const BigramDictContent *const sourceBigramDictContent, const int toPos, 184 const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, 185 int *const outEntrycount) { 186 bool hasNext = true; 187 int readingPos = bigramListPos; 188 int writingPos = toPos; 189 int lastEntryPos = NOT_A_DICT_POS; 190 while (hasNext) { 191 const BigramEntry originalBigramEntry = 192 sourceBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos); 193 hasNext = originalBigramEntry.hasNext(); 194 if (originalBigramEntry.getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) { 195 continue; 196 } 197 TerminalPositionLookupTable::TerminalIdMap::const_iterator it = 198 terminalIdMap->find(originalBigramEntry.getTargetTerminalId()); 199 if (it == terminalIdMap->end()) { 200 // Target word has been removed. 201 continue; 202 } 203 lastEntryPos = hasNext ? writingPos : NOT_A_DICT_POS; 204 const BigramEntry updatedBigramEntry = 205 originalBigramEntry.updateTargetTerminalIdAndGetEntry(it->second); 206 if (!writeBigramEntryAndAdvancePosition(&updatedBigramEntry, &writingPos)) { 207 AKLOGE("Cannot write bigram entry to run GC. pos: %d", writingPos); 208 return false; 209 } 210 *outEntrycount += 1; 211 } 212 if (lastEntryPos != NOT_A_DICT_POS) { 213 // Update has next flag in the last written entry. 214 const BigramEntry bigramEntry = getBigramEntry(lastEntryPos).updateHasNextAndGetEntry( 215 false /* hasNext */); 216 if (!writeBigramEntry(&bigramEntry, lastEntryPos)) { 217 AKLOGE("Cannot write bigram entry to set hasNext flag after GC. pos: %d", writingPos); 218 return false; 219 } 220 } 221 return true; 222 } 223 224 } // namespace v402 225 } // namespace backward 226 } // namespace latinime 227
