1 // Protocol Buffers - Google's data interchange format 2 // Copyright 2008 Google Inc. All rights reserved. 3 // https://developers.google.com/protocol-buffers/ 4 // 5 // Redistribution and use in source and binary forms, with or without 6 // modification, are permitted provided that the following conditions are 7 // met: 8 // 9 // * Redistributions of source code must retain the above copyright 10 // notice, this list of conditions and the following disclaimer. 11 // * Redistributions in binary form must reproduce the above 12 // copyright notice, this list of conditions and the following disclaimer 13 // in the documentation and/or other materials provided with the 14 // distribution. 15 // * Neither the name of Google Inc. nor the names of its 16 // contributors may be used to endorse or promote products derived from 17 // this software without specific prior written permission. 18 // 19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 // Author: brianolson (at) google.com (Brian Olson) 32 // 33 // This file contains the implementation of classes GzipInputStream and 34 // GzipOutputStream. 35 36 37 #if HAVE_ZLIB 38 #include <google/protobuf/io/gzip_stream.h> 39 40 #include <google/protobuf/stubs/common.h> 41 #include <google/protobuf/stubs/logging.h> 42 43 namespace google { 44 namespace protobuf { 45 namespace io { 46 47 static const int kDefaultBufferSize = 65536; 48 49 GzipInputStream::GzipInputStream( 50 ZeroCopyInputStream* sub_stream, Format format, int buffer_size) 51 : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) { 52 zcontext_.state = Z_NULL; 53 zcontext_.zalloc = Z_NULL; 54 zcontext_.zfree = Z_NULL; 55 zcontext_.opaque = Z_NULL; 56 zcontext_.total_out = 0; 57 zcontext_.next_in = NULL; 58 zcontext_.avail_in = 0; 59 zcontext_.total_in = 0; 60 zcontext_.msg = NULL; 61 if (buffer_size == -1) { 62 output_buffer_length_ = kDefaultBufferSize; 63 } else { 64 output_buffer_length_ = buffer_size; 65 } 66 output_buffer_ = operator new(output_buffer_length_); 67 GOOGLE_CHECK(output_buffer_ != NULL); 68 zcontext_.next_out = static_cast<Bytef*>(output_buffer_); 69 zcontext_.avail_out = output_buffer_length_; 70 output_position_ = output_buffer_; 71 } 72 GzipInputStream::~GzipInputStream() { 73 operator delete(output_buffer_); 74 zerror_ = inflateEnd(&zcontext_); 75 } 76 77 static inline int internalInflateInit2( 78 z_stream* zcontext, GzipInputStream::Format format) { 79 int windowBitsFormat = 0; 80 switch (format) { 81 case GzipInputStream::GZIP: windowBitsFormat = 16; break; 82 case GzipInputStream::AUTO: windowBitsFormat = 32; break; 83 case GzipInputStream::ZLIB: windowBitsFormat = 0; break; 84 } 85 return inflateInit2(zcontext, /* windowBits */15 | windowBitsFormat); 86 } 87 88 int GzipInputStream::Inflate(int flush) { 89 if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) { 90 // previous inflate filled output buffer. don't change input params yet. 91 } else if (zcontext_.avail_in == 0) { 92 const void* in; 93 int in_size; 94 bool first = zcontext_.next_in == NULL; 95 bool ok = sub_stream_->Next(&in, &in_size); 96 if (!ok) { 97 zcontext_.next_out = NULL; 98 zcontext_.avail_out = 0; 99 return Z_STREAM_END; 100 } 101 zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in)); 102 zcontext_.avail_in = in_size; 103 if (first) { 104 int error = internalInflateInit2(&zcontext_, format_); 105 if (error != Z_OK) { 106 return error; 107 } 108 } 109 } 110 zcontext_.next_out = static_cast<Bytef*>(output_buffer_); 111 zcontext_.avail_out = output_buffer_length_; 112 output_position_ = output_buffer_; 113 int error = inflate(&zcontext_, flush); 114 return error; 115 } 116 117 void GzipInputStream::DoNextOutput(const void** data, int* size) { 118 *data = output_position_; 119 *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_); 120 output_position_ = zcontext_.next_out; 121 } 122 123 // implements ZeroCopyInputStream ---------------------------------- 124 bool GzipInputStream::Next(const void** data, int* size) { 125 bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) 126 || (zerror_ == Z_BUF_ERROR); 127 if ((!ok) || (zcontext_.next_out == NULL)) { 128 return false; 129 } 130 if (zcontext_.next_out != output_position_) { 131 DoNextOutput(data, size); 132 return true; 133 } 134 if (zerror_ == Z_STREAM_END) { 135 if (zcontext_.next_out != NULL) { 136 // sub_stream_ may have concatenated streams to follow 137 zerror_ = inflateEnd(&zcontext_); 138 byte_count_ += zcontext_.total_out; 139 if (zerror_ != Z_OK) { 140 return false; 141 } 142 zerror_ = internalInflateInit2(&zcontext_, format_); 143 if (zerror_ != Z_OK) { 144 return false; 145 } 146 } else { 147 *data = NULL; 148 *size = 0; 149 return false; 150 } 151 } 152 zerror_ = Inflate(Z_NO_FLUSH); 153 if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) { 154 // The underlying stream's Next returned false inside Inflate. 155 return false; 156 } 157 ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) 158 || (zerror_ == Z_BUF_ERROR); 159 if (!ok) { 160 return false; 161 } 162 DoNextOutput(data, size); 163 return true; 164 } 165 void GzipInputStream::BackUp(int count) { 166 output_position_ = reinterpret_cast<void*>( 167 reinterpret_cast<uintptr_t>(output_position_) - count); 168 } 169 bool GzipInputStream::Skip(int count) { 170 const void* data; 171 int size; 172 bool ok = Next(&data, &size); 173 while (ok && (size < count)) { 174 count -= size; 175 ok = Next(&data, &size); 176 } 177 if (size > count) { 178 BackUp(size - count); 179 } 180 return ok; 181 } 182 int64 GzipInputStream::ByteCount() const { 183 int64 ret = byte_count_ + zcontext_.total_out; 184 if (zcontext_.next_out != NULL && output_position_ != NULL) { 185 ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) - 186 reinterpret_cast<uintptr_t>(output_position_); 187 } 188 return ret; 189 } 190 191 // ========================================================================= 192 193 GzipOutputStream::Options::Options() 194 : format(GZIP), 195 buffer_size(kDefaultBufferSize), 196 compression_level(Z_DEFAULT_COMPRESSION), 197 compression_strategy(Z_DEFAULT_STRATEGY) {} 198 199 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) { 200 Init(sub_stream, Options()); 201 } 202 203 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream, 204 const Options& options) { 205 Init(sub_stream, options); 206 } 207 208 void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream, 209 const Options& options) { 210 sub_stream_ = sub_stream; 211 sub_data_ = NULL; 212 sub_data_size_ = 0; 213 214 input_buffer_length_ = options.buffer_size; 215 input_buffer_ = operator new(input_buffer_length_); 216 GOOGLE_CHECK(input_buffer_ != NULL); 217 218 zcontext_.zalloc = Z_NULL; 219 zcontext_.zfree = Z_NULL; 220 zcontext_.opaque = Z_NULL; 221 zcontext_.next_out = NULL; 222 zcontext_.avail_out = 0; 223 zcontext_.total_out = 0; 224 zcontext_.next_in = NULL; 225 zcontext_.avail_in = 0; 226 zcontext_.total_in = 0; 227 zcontext_.msg = NULL; 228 // default to GZIP format 229 int windowBitsFormat = 16; 230 if (options.format == ZLIB) { 231 windowBitsFormat = 0; 232 } 233 zerror_ = deflateInit2( 234 &zcontext_, 235 options.compression_level, 236 Z_DEFLATED, 237 /* windowBits */15 | windowBitsFormat, 238 /* memLevel (default) */8, 239 options.compression_strategy); 240 } 241 242 GzipOutputStream::~GzipOutputStream() { 243 Close(); 244 operator delete(input_buffer_); 245 } 246 247 // private 248 int GzipOutputStream::Deflate(int flush) { 249 int error = Z_OK; 250 do { 251 if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) { 252 bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_); 253 if (!ok) { 254 sub_data_ = NULL; 255 sub_data_size_ = 0; 256 return Z_BUF_ERROR; 257 } 258 GOOGLE_CHECK_GT(sub_data_size_, 0); 259 zcontext_.next_out = static_cast<Bytef*>(sub_data_); 260 zcontext_.avail_out = sub_data_size_; 261 } 262 error = deflate(&zcontext_, flush); 263 } while (error == Z_OK && zcontext_.avail_out == 0); 264 if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) { 265 // Notify lower layer of data. 266 sub_stream_->BackUp(zcontext_.avail_out); 267 // We don't own the buffer anymore. 268 sub_data_ = NULL; 269 sub_data_size_ = 0; 270 } 271 return error; 272 } 273 274 // implements ZeroCopyOutputStream --------------------------------- 275 bool GzipOutputStream::Next(void** data, int* size) { 276 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) { 277 return false; 278 } 279 if (zcontext_.avail_in != 0) { 280 zerror_ = Deflate(Z_NO_FLUSH); 281 if (zerror_ != Z_OK) { 282 return false; 283 } 284 } 285 if (zcontext_.avail_in == 0) { 286 // all input was consumed. reset the buffer. 287 zcontext_.next_in = static_cast<Bytef*>(input_buffer_); 288 zcontext_.avail_in = input_buffer_length_; 289 *data = input_buffer_; 290 *size = input_buffer_length_; 291 } else { 292 // The loop in Deflate should consume all avail_in 293 GOOGLE_LOG(DFATAL) << "Deflate left bytes unconsumed"; 294 } 295 return true; 296 } 297 void GzipOutputStream::BackUp(int count) { 298 GOOGLE_CHECK_GE(zcontext_.avail_in, count); 299 zcontext_.avail_in -= count; 300 } 301 int64 GzipOutputStream::ByteCount() const { 302 return zcontext_.total_in + zcontext_.avail_in; 303 } 304 305 bool GzipOutputStream::Flush() { 306 zerror_ = Deflate(Z_FULL_FLUSH); 307 // Return true if the flush succeeded or if it was a no-op. 308 return (zerror_ == Z_OK) || 309 (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 && 310 zcontext_.avail_out != 0); 311 } 312 313 bool GzipOutputStream::Close() { 314 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) { 315 return false; 316 } 317 do { 318 zerror_ = Deflate(Z_FINISH); 319 } while (zerror_ == Z_OK); 320 zerror_ = deflateEnd(&zcontext_); 321 bool ok = zerror_ == Z_OK; 322 zerror_ = Z_STREAM_END; 323 return ok; 324 } 325 326 } // namespace io 327 } // namespace protobuf 328 } // namespace google 329 330 #endif // HAVE_ZLIB 331
