1 /*------------------------------------------------------------------------- 2 * Vulkan CTS Framework 3 * -------------------- 4 * 5 * Copyright (c) 2015 Google Inc. 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 * 19 *//*! 20 * \file 21 * \brief Program binary registry. 22 *//*--------------------------------------------------------------------*/ 23 24 #include "vkBinaryRegistry.hpp" 25 #include "tcuResource.hpp" 26 #include "tcuFormatUtil.hpp" 27 #include "deFilePath.hpp" 28 #include "deStringUtil.hpp" 29 #include "deString.h" 30 #include "deInt32.h" 31 32 #include <sstream> 33 #include <fstream> 34 #include <stdexcept> 35 #include <limits> 36 37 namespace vk 38 { 39 namespace BinaryRegistryDetail 40 { 41 42 using std::string; 43 using std::vector; 44 45 namespace 46 { 47 48 string getProgramPath (const std::string& dirName, deUint32 index) 49 { 50 return de::FilePath::join(dirName, de::toString(tcu::toHex(index)) + ".spv").getPath(); 51 } 52 53 string getIndexPath (const std::string& dirName) 54 { 55 return de::FilePath::join(dirName, "index.bin").getPath(); 56 } 57 58 void writeBinary (const std::string& dstDir, deUint32 index, const ProgramBinary& binary) 59 { 60 const de::FilePath fullPath = getProgramPath(dstDir, index); 61 62 if (!de::FilePath(fullPath.getDirName()).exists()) 63 de::createDirectoryAndParents(fullPath.getDirName().c_str()); 64 65 { 66 std::ofstream out (fullPath.getPath(), std::ios_base::binary); 67 68 if (!out.is_open() || !out.good()) 69 throw tcu::Exception("Failed to open " + string(fullPath.getPath())); 70 71 out.write((const char*)binary.getBinary(), binary.getSize()); 72 out.close(); 73 } 74 } 75 76 deUint32 binaryHash (const ProgramBinary* binary) 77 { 78 return deMemoryHash(binary->getBinary(), binary->getSize()); 79 } 80 81 deBool binaryEqual (const ProgramBinary* a, const ProgramBinary* b) 82 { 83 if (a->getSize() == b->getSize()) 84 return deMemoryEqual(a->getBinary(), b->getBinary(), a->getSize()); 85 else 86 return DE_FALSE; 87 } 88 89 std::vector<deUint32> getSearchPath (const ProgramIdentifier& id) 90 { 91 const std::string combinedStr = id.testCasePath + '#' + id.programName; 92 const size_t strLen = combinedStr.size(); 93 const size_t numWords = strLen/4 + 1; // Must always end up with at least one 0 byte 94 vector<deUint32> words (numWords, 0u); 95 96 deMemcpy(&words[0], combinedStr.c_str(), strLen); 97 98 return words; 99 } 100 101 const deUint32* findBinaryIndex (BinaryIndexAccess* index, const ProgramIdentifier& id) 102 { 103 const vector<deUint32> words = getSearchPath(id); 104 size_t nodeNdx = 0; 105 size_t wordNdx = 0; 106 107 for (;;) 108 { 109 const BinaryIndexNode& curNode = (*index)[nodeNdx]; 110 111 if (curNode.word == words[wordNdx]) 112 { 113 if (wordNdx+1 < words.size()) 114 { 115 TCU_CHECK_INTERNAL((size_t)curNode.index < index->size()); 116 117 nodeNdx = curNode.index; 118 wordNdx += 1; 119 } 120 else if (wordNdx+1 == words.size()) 121 return &curNode.index; 122 else 123 return DE_NULL; 124 } 125 else if (curNode.word != 0) 126 { 127 nodeNdx += 1; 128 129 // Index should always be null-terminated 130 TCU_CHECK_INTERNAL(nodeNdx < index->size()); 131 } 132 else 133 return DE_NULL; 134 } 135 136 return DE_NULL; 137 } 138 139 //! Sparse index node used for final binary index construction 140 struct SparseIndexNode 141 { 142 deUint32 word; 143 deUint32 index; 144 std::vector<SparseIndexNode*> children; 145 146 SparseIndexNode (deUint32 word_, deUint32 index_) 147 : word (word_) 148 , index (index_) 149 {} 150 151 SparseIndexNode (void) 152 : word (0) 153 , index (0) 154 {} 155 156 ~SparseIndexNode (void) 157 { 158 for (size_t ndx = 0; ndx < children.size(); ndx++) 159 delete children[ndx]; 160 } 161 }; 162 163 #if defined(DE_DEBUG) 164 bool isNullByteTerminated (deUint32 word) 165 { 166 deUint8 bytes[4]; 167 deMemcpy(bytes, &word, sizeof(word)); 168 return bytes[3] == 0; 169 } 170 #endif 171 172 void addToSparseIndex (SparseIndexNode* group, const deUint32* words, size_t numWords, deUint32 index) 173 { 174 const deUint32 curWord = words[0]; 175 SparseIndexNode* child = DE_NULL; 176 177 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++) 178 { 179 if (group->children[childNdx]->word == curWord) 180 { 181 child = group->children[childNdx]; 182 break; 183 } 184 } 185 186 DE_ASSERT(numWords > 1 || !child); 187 188 if (!child) 189 { 190 group->children.reserve(group->children.size()+1); 191 group->children.push_back(new SparseIndexNode(curWord, numWords == 1 ? index : 0)); 192 193 child = group->children.back(); 194 } 195 196 if (numWords > 1) 197 addToSparseIndex(child, words+1, numWords-1, index); 198 else 199 DE_ASSERT(isNullByteTerminated(curWord)); 200 } 201 202 // Prepares sparse index for finalization. Ensures that child with word = 0 is moved 203 // to the end, or one is added if there is no such child already. 204 void normalizeSparseIndex (SparseIndexNode* group) 205 { 206 int zeroChildPos = -1; 207 208 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++) 209 { 210 normalizeSparseIndex(group->children[childNdx]); 211 212 if (group->children[childNdx]->word == 0) 213 { 214 DE_ASSERT(zeroChildPos < 0); 215 zeroChildPos = (int)childNdx; 216 } 217 } 218 219 if (zeroChildPos >= 0) 220 { 221 // Move child with word = 0 to last 222 while (zeroChildPos != (int)group->children.size()-1) 223 { 224 std::swap(group->children[zeroChildPos], group->children[zeroChildPos+1]); 225 zeroChildPos += 1; 226 } 227 } 228 else if (!group->children.empty()) 229 { 230 group->children.reserve(group->children.size()+1); 231 group->children.push_back(new SparseIndexNode(0, 0)); 232 } 233 } 234 235 deUint32 getIndexSize (const SparseIndexNode* group) 236 { 237 size_t numNodes = group->children.size(); 238 239 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++) 240 numNodes += getIndexSize(group->children[childNdx]); 241 242 DE_ASSERT(numNodes <= std::numeric_limits<deUint32>::max()); 243 244 return (deUint32)numNodes; 245 } 246 247 deUint32 addAndCountNodes (BinaryIndexNode* index, deUint32 baseOffset, const SparseIndexNode* group) 248 { 249 const deUint32 numLocalNodes = (deUint32)group->children.size(); 250 deUint32 curOffset = numLocalNodes; 251 252 // Must be normalized prior to construction of final index 253 DE_ASSERT(group->children.empty() || group->children.back()->word == 0); 254 255 for (size_t childNdx = 0; childNdx < numLocalNodes; childNdx++) 256 { 257 const SparseIndexNode* child = group->children[childNdx]; 258 const deUint32 subtreeSize = addAndCountNodes(index+curOffset, baseOffset+curOffset, child); 259 260 index[childNdx].word = child->word; 261 262 if (subtreeSize == 0) 263 index[childNdx].index = child->index; 264 else 265 { 266 DE_ASSERT(child->index == 0); 267 index[childNdx].index = baseOffset+curOffset; 268 } 269 270 curOffset += subtreeSize; 271 } 272 273 return curOffset; 274 } 275 276 void buildFinalIndex (std::vector<BinaryIndexNode>* dst, const SparseIndexNode* root) 277 { 278 const deUint32 indexSize = getIndexSize(root); 279 280 if (indexSize > 0) 281 { 282 dst->resize(indexSize); 283 addAndCountNodes(&(*dst)[0], 0, root); 284 } 285 else 286 { 287 // Generate empty index 288 dst->resize(1); 289 (*dst)[0].word = 0u; 290 (*dst)[0].index = 0u; 291 } 292 } 293 294 } // anonymous 295 296 // BinaryRegistryWriter 297 298 DE_IMPLEMENT_POOL_HASH(BinaryHash, const ProgramBinary*, deUint32, binaryHash, binaryEqual); 299 300 BinaryRegistryWriter::BinaryRegistryWriter (const std::string& dstPath) 301 : m_dstPath (dstPath) 302 , m_binaryIndexMap (DE_NULL) 303 { 304 m_binaryIndexMap = BinaryHash_create(m_memPool.getRawPool()); 305 306 if (!m_binaryIndexMap) 307 throw std::bad_alloc(); 308 } 309 310 BinaryRegistryWriter::~BinaryRegistryWriter (void) 311 { 312 for (BinaryVector::const_iterator binaryIter = m_compactedBinaries.begin(); 313 binaryIter != m_compactedBinaries.end(); 314 ++binaryIter) 315 delete *binaryIter; 316 } 317 318 void BinaryRegistryWriter::storeProgram (const ProgramIdentifier& id, const ProgramBinary& binary) 319 { 320 const deUint32* const indexPtr = BinaryHash_find(m_binaryIndexMap, &binary); 321 deUint32 index = indexPtr ? *indexPtr : ~0u; 322 323 DE_ASSERT(binary.getFormat() == vk::PROGRAM_FORMAT_SPIRV); 324 325 if (!indexPtr) 326 { 327 ProgramBinary* const binaryClone = new ProgramBinary(binary); 328 329 try 330 { 331 index = (deUint32)m_compactedBinaries.size(); 332 m_compactedBinaries.push_back(binaryClone); 333 } 334 catch (...) 335 { 336 delete binaryClone; 337 throw; 338 } 339 340 writeBinary(m_dstPath, index, binary); 341 342 if (!BinaryHash_insert(m_binaryIndexMap, binaryClone, index)) 343 throw std::bad_alloc(); 344 } 345 346 DE_ASSERT((size_t)index < m_compactedBinaries.size()); 347 348 m_binaryIndices.push_back(BinaryIndex(id, index)); 349 } 350 351 void BinaryRegistryWriter::writeIndex (void) const 352 { 353 const de::FilePath indexPath = getIndexPath(m_dstPath); 354 std::vector<BinaryIndexNode> index; 355 356 { 357 de::UniquePtr<SparseIndexNode> sparseIndex (new SparseIndexNode()); 358 359 for (size_t progNdx = 0; progNdx < m_binaryIndices.size(); progNdx++) 360 { 361 const std::vector<deUint32> searchPath = getSearchPath(m_binaryIndices[progNdx].id); 362 addToSparseIndex(sparseIndex.get(), &searchPath[0], searchPath.size(), m_binaryIndices[progNdx].index); 363 } 364 365 normalizeSparseIndex(sparseIndex.get()); 366 buildFinalIndex(&index, sparseIndex.get()); 367 } 368 369 // Even in empty index there is always terminating node for the root group 370 DE_ASSERT(!index.empty()); 371 372 if (!de::FilePath(indexPath.getDirName()).exists()) 373 de::createDirectoryAndParents(indexPath.getDirName().c_str()); 374 375 { 376 std::ofstream indexOut(indexPath.getPath(), std::ios_base::binary); 377 378 if (!indexOut.is_open() || !indexOut.good()) 379 throw tcu::InternalError(string("Failed to open program binary index file ") + indexPath.getPath()); 380 381 indexOut.write((const char*)&index[0], index.size()*sizeof(BinaryIndexNode)); 382 } 383 } 384 385 // BinaryRegistryReader 386 387 BinaryRegistryReader::BinaryRegistryReader (const tcu::Archive& archive, const std::string& srcPath) 388 : m_archive (archive) 389 , m_srcPath (srcPath) 390 { 391 } 392 393 BinaryRegistryReader::~BinaryRegistryReader (void) 394 { 395 } 396 397 ProgramBinary* BinaryRegistryReader::loadProgram (const ProgramIdentifier& id) const 398 { 399 if (!m_binaryIndex) 400 { 401 try 402 { 403 m_binaryIndex = BinaryIndexPtr(new BinaryIndexAccess(de::MovePtr<tcu::Resource>(m_archive.getResource(getIndexPath(m_srcPath).c_str())))); 404 } 405 catch (const tcu::ResourceError& e) 406 { 407 throw ProgramNotFoundException(id, string("Failed to open binary index (") + e.what() + ")"); 408 } 409 } 410 411 { 412 const deUint32* indexPos = findBinaryIndex(m_binaryIndex.get(), id); 413 414 if (indexPos) 415 { 416 const string fullPath = getProgramPath(m_srcPath, *indexPos); 417 418 try 419 { 420 de::UniquePtr<tcu::Resource> progRes (m_archive.getResource(fullPath.c_str())); 421 const int progSize = progRes->getSize(); 422 vector<deUint8> bytes (progSize); 423 424 TCU_CHECK_INTERNAL(!bytes.empty()); 425 426 progRes->read(&bytes[0], progSize); 427 428 return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]); 429 } 430 catch (const tcu::ResourceError& e) 431 { 432 throw ProgramNotFoundException(id, e.what()); 433 } 434 } 435 else 436 throw ProgramNotFoundException(id, "Program not found in index"); 437 } 438 } 439 440 } // BinaryRegistryDetail 441 } // vk 442
