1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include "tensorflow/core/lib/io/format.h" 17 18 #include "tensorflow/core/lib/core/coding.h" 19 #include "tensorflow/core/lib/core/errors.h" 20 #include "tensorflow/core/lib/hash/crc32c.h" 21 #include "tensorflow/core/lib/io/block.h" 22 #include "tensorflow/core/platform/env.h" 23 #include "tensorflow/core/platform/snappy.h" 24 25 namespace tensorflow { 26 namespace table { 27 28 void BlockHandle::EncodeTo(string* dst) const { 29 // Sanity check that all fields have been set 30 assert(offset_ != ~static_cast<uint64>(0)); 31 assert(size_ != ~static_cast<uint64>(0)); 32 core::PutVarint64(dst, offset_); 33 core::PutVarint64(dst, size_); 34 } 35 36 Status BlockHandle::DecodeFrom(StringPiece* input) { 37 if (core::GetVarint64(input, &offset_) && core::GetVarint64(input, &size_)) { 38 return Status::OK(); 39 } else { 40 return errors::DataLoss("bad block handle"); 41 } 42 } 43 44 void Footer::EncodeTo(string* dst) const { 45 #ifndef NDEBUG 46 const size_t original_size = dst->size(); 47 #endif 48 metaindex_handle_.EncodeTo(dst); 49 index_handle_.EncodeTo(dst); 50 dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding 51 core::PutFixed32(dst, static_cast<uint32>(kTableMagicNumber & 0xffffffffu)); 52 core::PutFixed32(dst, static_cast<uint32>(kTableMagicNumber >> 32)); 53 assert(dst->size() == original_size + kEncodedLength); 54 } 55 56 Status Footer::DecodeFrom(StringPiece* input) { 57 const char* magic_ptr = input->data() + kEncodedLength - 8; 58 const uint32 magic_lo = core::DecodeFixed32(magic_ptr); 59 const uint32 magic_hi = core::DecodeFixed32(magic_ptr + 4); 60 const uint64 magic = 61 ((static_cast<uint64>(magic_hi) << 32) | (static_cast<uint64>(magic_lo))); 62 if (magic != kTableMagicNumber) { 63 return errors::DataLoss("not an sstable (bad magic number)"); 64 } 65 66 Status result = metaindex_handle_.DecodeFrom(input); 67 if (result.ok()) { 68 result = index_handle_.DecodeFrom(input); 69 } 70 if (result.ok()) { 71 // We skip over any leftover data (just padding for now) in "input" 72 const char* end = magic_ptr + 8; 73 *input = StringPiece(end, input->data() + input->size() - end); 74 } 75 return result; 76 } 77 78 Status ReadBlock(RandomAccessFile* file, const BlockHandle& handle, 79 BlockContents* result) { 80 result->data = StringPiece(); 81 result->cachable = false; 82 result->heap_allocated = false; 83 84 // Read the block contents as well as the type/crc footer. 85 // See table_builder.cc for the code that built this structure. 86 size_t n = static_cast<size_t>(handle.size()); 87 char* buf = new char[n + kBlockTrailerSize]; 88 StringPiece contents; 89 Status s = file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf); 90 if (!s.ok()) { 91 delete[] buf; 92 return s; 93 } 94 if (contents.size() != n + kBlockTrailerSize) { 95 delete[] buf; 96 return errors::DataLoss("truncated block read"); 97 } 98 99 // Check the crc of the type and the block contents 100 const char* data = contents.data(); // Pointer to where Read put the data 101 // This checksum verification is optional. We leave it on for now 102 const bool verify_checksum = true; 103 if (verify_checksum) { 104 const uint32 crc = crc32c::Unmask(core::DecodeFixed32(data + n + 1)); 105 const uint32 actual = crc32c::Value(data, n + 1); 106 if (actual != crc) { 107 delete[] buf; 108 s = errors::DataLoss("block checksum mismatch"); 109 return s; 110 } 111 } 112 113 switch (data[n]) { 114 case kNoCompression: 115 if (data != buf) { 116 // File implementation gave us pointer to some other data. 117 // Use it directly under the assumption that it will be live 118 // while the file is open. 119 delete[] buf; 120 result->data = StringPiece(data, n); 121 result->heap_allocated = false; 122 result->cachable = false; // Do not double-cache 123 } else { 124 result->data = StringPiece(buf, n); 125 result->heap_allocated = true; 126 result->cachable = true; 127 } 128 129 // Ok 130 break; 131 case kSnappyCompression: { 132 size_t ulength = 0; 133 if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) { 134 delete[] buf; 135 return errors::DataLoss("corrupted compressed block contents"); 136 } 137 char* ubuf = new char[ulength]; 138 if (!port::Snappy_Uncompress(data, n, ubuf)) { 139 delete[] buf; 140 delete[] ubuf; 141 return errors::DataLoss("corrupted compressed block contents"); 142 } 143 delete[] buf; 144 result->data = StringPiece(ubuf, ulength); 145 result->heap_allocated = true; 146 result->cachable = true; 147 break; 148 } 149 default: 150 delete[] buf; 151 return errors::DataLoss("bad block type"); 152 } 153 154 return Status::OK(); 155 } 156 157 } // namespace table 158 } // namespace tensorflow 159
