1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_LIB_IO_INPUTBUFFER_H_ 17 #define TENSORFLOW_LIB_IO_INPUTBUFFER_H_ 18 19 #include <string> 20 #include "tensorflow/core/lib/core/coding.h" 21 #include "tensorflow/core/lib/core/status.h" 22 #include "tensorflow/core/platform/env.h" 23 #include "tensorflow/core/platform/macros.h" 24 #include "tensorflow/core/platform/types.h" 25 26 namespace tensorflow { 27 namespace io { 28 29 // An InputBuffer provides a buffer on top of a RandomAccessFile. 30 // A given instance of an InputBuffer is NOT safe for concurrent use 31 // by multiple threads 32 class InputBuffer { 33 public: 34 // Create an InputBuffer for "file" with a buffer size of 35 // "buffer_bytes" bytes. 'file' must outlive *this. 36 InputBuffer(RandomAccessFile* file, size_t buffer_bytes); 37 ~InputBuffer(); 38 39 // Read one text line of data into "*result" until end-of-file or a 40 // \n is read. (The \n is not included in the result.) Overwrites 41 // any existing data in *result. 42 // 43 // If successful, returns OK. If we are already at the end of the 44 // file, we return an OUT_OF_RANGE error. Otherwise, we return 45 // some other non-OK status. 46 Status ReadLine(string* result); 47 48 // Reads bytes_to_read bytes into *result, overwriting *result. 49 // 50 // If successful, returns OK. If we there are not enough bytes to 51 // read before the end of the file, we return an OUT_OF_RANGE error. 52 // Otherwise, we return some other non-OK status. 53 Status ReadNBytes(int64 bytes_to_read, string* result); 54 55 // An overload that writes to char*. Caller must ensure result[0, 56 // bytes_to_read) is valid to be overwritten. Returns OK iff "*bytes_read == 57 // bytes_to_read". 58 Status ReadNBytes(int64 bytes_to_read, char* result, size_t* bytes_read); 59 60 // Reads a single varint32. 61 Status ReadVarint32(uint32* result); 62 63 // Reads a single varint64. 64 Status ReadVarint64(uint64* result); 65 66 // Like ReadNBytes() without returning the bytes read. 67 Status SkipNBytes(int64 bytes_to_skip); 68 69 // Seek to this offset within the file. 70 // 71 // If we seek to somewhere within our pre-buffered data, we will re-use what 72 // data we can. Otherwise, Seek() throws out the current buffer and the next 73 // read will trigger a File::Read(). 74 Status Seek(int64 position); 75 76 // Returns the position in the file. 77 int64 Tell() const { return file_pos_ - (limit_ - pos_); } 78 79 // Returns the underlying RandomAccessFile. 80 RandomAccessFile* file() const { return file_; } 81 82 private: 83 Status FillBuffer(); 84 85 // Internal slow-path routine used by ReadVarint32(). 86 Status ReadVarint32Fallback(uint32* result); 87 88 // Internal slow-path routine used by ReadVarint64(). 89 Status ReadVarint64Fallback(uint64* result); 90 91 // Helper method for reading a varint which can span at max `max_bytes`. 92 // If the varint is longer, a DataLoss error status is returned. 93 // If end of file is reached while reading, OutOfRange error is returned. 94 template <typename T> 95 Status ReadVarintFallback(T* result, int max_bytes); 96 97 RandomAccessFile* file_; // Not owned 98 int64 file_pos_; // Next position to read from in "file_" 99 size_t size_; // Size of "buf_" 100 char* buf_; // The buffer itself 101 // [pos_,limit_) hold the "limit_ - pos_" bytes just before "file_pos_" 102 char* pos_; // Current position in "buf" 103 char* limit_; // Just past end of valid data in "buf" 104 105 TF_DISALLOW_COPY_AND_ASSIGN(InputBuffer); 106 }; 107 108 // Implementation details. 109 110 // Inlined for performance. 111 inline Status InputBuffer::ReadVarint32(uint32* result) { 112 if (pos_ + core::kMaxVarint32Bytes <= limit_) { 113 // Fast path: directly parse from buffered data. 114 // Reads strictly from the range [pos_, limit_). 115 const char* offset = core::GetVarint32Ptr(pos_, limit_, result); 116 if (offset == nullptr) return errors::OutOfRange("Parsed past limit."); 117 pos_ = const_cast<char*>(offset); 118 return Status::OK(); 119 } else { 120 return ReadVarint32Fallback(result); 121 } 122 } 123 124 // Inlined for performance. 125 inline Status InputBuffer::ReadVarint64(uint64* result) { 126 if (pos_ + core::kMaxVarint64Bytes <= limit_) { 127 // Fast path: directly parse from buffered data. 128 // Reads strictly from the range [pos_, limit_). 129 const char* offset = core::GetVarint64Ptr(pos_, limit_, result); 130 if (offset == nullptr) return errors::OutOfRange("Parsed past limit."); 131 pos_ = const_cast<char*>(offset); 132 return Status::OK(); 133 } else { 134 return ReadVarint64Fallback(result); 135 } 136 } 137 138 } // namespace io 139 } // namespace tensorflow 140 141 #endif // TENSORFLOW_LIB_IO_INPUTBUFFER_H_ 142
