open-vcdiff/src/vcdiff_main.cc

// Copyright 2008 Google Inc.
// Author: Lincoln Smith
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// A command-line interface to the open-vcdiff library.

#include <config.h>
#include <assert.h>
#include <errno.h>
#ifdef WIN32
#include <fcntl.h>
#include <io.h>
#endif  // WIN32
#include <stdio.h>
#include <string.h>  // strerror
#include <iostream>
#include <memory>
#include <string>
#include <vector>
#include "gflags/gflags.h"
#include "google/vcdecoder.h"
#include "google/vcencoder.h"

#ifndef HAS_GLOBAL_STRING
using std::string;
#endif  // !HAS_GLOBAL_STRING
using google::GetCommandLineFlagInfoOrDie;
using google::ShowUsageWithFlagsRestrict;

static const size_t kDefaultMaxTargetSize = 1 << 26;      // 64 MB

// Definitions of command-line flags
DEFINE_string(dictionary, "",
              "File containing dictionary data (required)");
DEFINE_string(target, "",
              "Target file (default is stdin for encode, stdout for decode");
DEFINE_string(delta, "",
              "Encoded delta file (default is stdout for encode, "
              "stdin for decode");
// --buffersize is the maximum allowable size of a target window.
// This value may be increased if there is sufficient memory available.
DEFINE_uint64(buffersize, 1 << 20,  // 1 MB
              "Buffer size for reading input file");
DEFINE_bool(allow_vcd_target, true,
            "If false, the decoder issues an error when the VCD_TARGET flag "
            "is encountered");
DEFINE_bool(checksum, false,
            "Include an Adler32 checksum of the target data when encoding");
DEFINE_bool(interleaved, false, "Use interleaved format");
DEFINE_bool(json, false, "Output diff in the JSON format when encoding");
DEFINE_bool(stats, false, "Report compression percentage");
DEFINE_bool(target_matches, false, "Find duplicate strings in target data"
                                   " as well as dictionary data");
DEFINE_uint64(max_target_file_size, kDefaultMaxTargetSize,
              "Maximum target file size allowed by decoder");
DEFINE_uint64(max_target_window_size, kDefaultMaxTargetSize,
              "Maximum target window size allowed by decoder");

static const char* const kUsageString =
    " {encode | delta | decode | patch }[ <options> ]\n"
    "encode or delta: create delta file from dictionary and target file\n"
    "decode or patch: reconstruct target file from dictionary and delta file";

namespace open_vcdiff {

class VCDiffFileBasedCoder {
 public:
  VCDiffFileBasedCoder();
  ~VCDiffFileBasedCoder();

  // Once the command-line arguments have been parsed, these functions
  // will use the supplied options to carry out a file-based encode
  // or decode operation.
  bool Encode();
  bool Decode();
  bool DecodeAndCompare();  // for "vcdiff test"; compare target with original

 private:
  // Determines the size of the file.  The given file must be an input file
  // opened for reading only, not an input stream such as stdin.  The function
  // returns true and populates file_size if successful; otherwise, it returns
  // false.
  static bool FileSize(FILE* file, size_t* file_size);

  // Opens a file for incremental reading.  file_name is the name of the file
  // to be opened.  file_type should be a descriptive name (like "target") for
  // use in log messages.  If successful, returns true and sets *file to a
  // valid input file, *buffer to a region of memory allocated using malloc()
  // (so the caller must release it using free()), and buffer_size to the size
  // of the buffer, which will not be larger than the size of the file, and
  // will not be smaller than the --buffersize option.  If the function fails,
  // it outputs a log message and returns false.
  bool OpenFileForReading(const string& file_name,
                          const char* file_type,
                          FILE** file,
                          std::vector<char>* buffer);

  // Opens the dictionary file and reads it into a newly allocated buffer.
  // If successful, returns true and populates dictionary_ with the dictionary
  // contents; otherwise, returns false.
  bool OpenDictionary();

  // Opens the input file (the delta or target file) for reading.
  // Allocates space for the input buffer.  If successful,
  // input_file_ will be valid and input_buffer_ will be allocated.
  bool OpenInputFile() {
    return OpenFileForReading(input_file_name_,
                              input_file_type_,
                              &input_file_,
                              &input_buffer_);
  }

  // Opens the output file (the target or delta file) for writing.
  // If successful, output_file_ will be valid.
  bool OpenOutputFile();

  // Opens the output file (the target file) for comparison against the decoded
  // output when using "vcdiff test".
  bool OpenOutputFileForCompare() {
    return OpenFileForReading(output_file_name_,
                              output_file_type_,
                              &output_file_,
                              &compare_buffer_);
  }

  // Reads as much input data as possible from the input file
  // into input_buffer_.  If successful, returns true and sets *bytes_read
  // to the number of bytes read into input_buffer_.  If an error occurs,
  // writes an error log message and returns false.
  bool ReadInput(size_t* bytes_read);

  // Writes the contents of output to output_file_.  If successful, returns
  // true.  If an error occurs, writes an error log message and returns false.
  bool WriteOutput(const string& output);

  // Reads a number of bytes from output_file_ equal to the size of output,
  // and compares to make sure they match the contents of output.  If the bytes
  // do not match, or if end of file is reached before the expected number of
  // bytes have been read, or a read error occurs, the function returns false;
  // otherwise, returns true.
  bool CompareOutput(const string& output);

  // Dictionary contents.  The entire dictionary file will be read into memory.
  std::vector<char> dictionary_;

  std::auto_ptr<open_vcdiff::HashedDictionary> hashed_dictionary_;

  // These should be set to either "delta" or "target".  They are only
  // used in log messages such as "Error opening delta file..."
  const char* input_file_type_;
  const char* output_file_type_;

  // The filenames used for input and output.  Will be empty if stdin
  // or stdout is being used.
  string input_file_name_;
  string output_file_name_;

  // stdio-style file handles for the input and output files and the dictionary.
  // When encoding, input_file_ is the target file and output_file_ is the delta
  // file; when decoding, the reverse is true.  The dictionary is always read
  // from a file rather than from standard input.
  FILE* input_file_;
  FILE* output_file_;

  // A memory buffer used to load the input file into memory.  If the input
  // comes from stdin because no input file was specified, then the size of
  // input_buffer_ will be the value specified by the --buffersize option.
  // If the input comes from a file, then the buffer will be allocated to match
  // the file size, if possible.  However, the buffer will not exceed
  // --buffersize bytes in length.
  std::vector<char> input_buffer_;

  // A memory buffer used to load the output file into memory for comparison
  // if "vcdiff test" is specified.
  std::vector<char> compare_buffer_;

  // Making these private avoids implicit copy constructor & assignment operator
  VCDiffFileBasedCoder(const VCDiffFileBasedCoder&);  // NOLINT
  void operator=(const VCDiffFileBasedCoder&);
};

inline VCDiffFileBasedCoder::VCDiffFileBasedCoder()
    : input_file_type_(""),
      output_file_type_(""),
      input_file_(NULL),
      output_file_(NULL) { }

VCDiffFileBasedCoder::~VCDiffFileBasedCoder() {
  if (input_file_ && (input_file_ != stdin)) {
    fclose(input_file_);
    input_file_ = NULL;
  }
  if (output_file_ && (output_file_ != stdout)) {
    fclose(output_file_);
    output_file_ = NULL;
  }
}

bool VCDiffFileBasedCoder::FileSize(FILE* file, size_t* file_size) {
  long initial_position = ftell(file);
  if (fseek(file, 0, SEEK_END) != 0) {
    return false;
  }
  *file_size = static_cast<size_t>(ftell(file));
  if (fseek(file, initial_position, SEEK_SET) != 0) {
    return false;
  }
  return true;
}

bool VCDiffFileBasedCoder::OpenDictionary() {
  assert(dictionary_.empty());
  assert(!FLAGS_dictionary.empty());
  FILE* dictionary_file = fopen(FLAGS_dictionary.c_str(), "rb");
  if (!dictionary_file) {
    std::cerr << "Error opening dictionary file '" << FLAGS_dictionary
              << "': " << strerror(errno) << std::endl;
    return false;
  }
  size_t dictionary_size = 0U;
  if (!FileSize(dictionary_file, &dictionary_size)) {
    std::cerr << "Error finding size of dictionary file '" << FLAGS_dictionary
              << "': " << strerror(errno) << std::endl;
    return false;
  }
  dictionary_.resize(dictionary_size);
  if (dictionary_size > 0) {
    if (fread(&dictionary_[0], 1, dictionary_size, dictionary_file)
            != dictionary_size) {
      std::cerr << "Unable to read dictionary file '" << FLAGS_dictionary
                << "': " << strerror(errno) << std::endl;
      fclose(dictionary_file);
      dictionary_.clear();
      return false;
    }
  }
  fclose(dictionary_file);
  return true;
}

bool VCDiffFileBasedCoder::OpenFileForReading(const string& file_name,
                                              const char* file_type,
                                              FILE** file,
                                              std::vector<char>* buffer) {
  assert(buffer->empty());
  size_t buffer_size = 0U;
  if (!*file && file_name.empty()) {
#ifdef WIN32
    _setmode(_fileno(stdin), _O_BINARY);
#endif
    *file = stdin;
    buffer_size = static_cast<size_t>(FLAGS_buffersize);
  } else {
    if (!*file) {
      *file = fopen(file_name.c_str(), "rb");
      if (!*file) {
        std::cerr << "Error opening " << file_type << " file '"
                  << file_name << "': " << strerror(errno) << std::endl;
        return false;
      }
    }
    size_t file_size = 0U;
    if (!FileSize(*file, &file_size)) {
      std::cerr << "Error finding size of " << file_type << " file '"
                << file_name << "': " << strerror(errno) << std::endl;
      return false;
    }
    buffer_size = static_cast<size_t>(FLAGS_buffersize);
    if (file_size < buffer_size) {
      // Allocate just enough memory to store the entire file
      buffer_size = file_size;
    }
  }
  buffer->resize(buffer_size);
  return true;
}

// Opens the output file for streamed read operations using the
// standard C I/O library, i.e., fopen(), fwrite(), fclose().
// No output buffer is allocated because the encoded/decoded output
// is constructed progressively using a std::string object
// whose buffer is resized as needed.
bool VCDiffFileBasedCoder::OpenOutputFile() {
  if (output_file_name_.empty()) {
#ifdef WIN32
    _setmode(_fileno(stdout), _O_BINARY);
#endif
    output_file_ = stdout;
  } else {
    output_file_ = fopen(output_file_name_.c_str(), "wb");
    if (!output_file_) {
      std::cerr << "Error opening " << output_file_type_ << " file '"
                << output_file_name_
                << "': " << strerror(errno) << std::endl;
      return false;
    }
  }
  return true;
}

bool VCDiffFileBasedCoder::ReadInput(size_t* bytes_read) {
  // Read from file or stdin
  *bytes_read = fread(&input_buffer_[0], 1, input_buffer_.size(), input_file_);
  if (ferror(input_file_)) {
    std::cerr << "Error reading from " << input_file_type_ << " file '"
              << input_file_name_
              << "': " << strerror(errno) << std::endl;
    return false;
  }
  return true;
}

bool VCDiffFileBasedCoder::WriteOutput(const string& output) {
  if (!output.empty()) {
    // Some new output has been generated and is ready to be written
    // to the output file or to stdout.
    fwrite(output.data(), 1, output.size(), output_file_);
    if (ferror(output_file_)) {
      std::cerr << "Error writing " << output.size() << " bytes to "
                << output_file_type_ << " file '" << output_file_name_
                << "': " << strerror(errno) << std::endl;
      return false;
    }
  }
  return true;
}

bool VCDiffFileBasedCoder::CompareOutput(const string& output) {
  if (!output.empty()) {
    size_t output_size = output.size();
    // Some new output has been generated and is ready to be compared against
    // the output file.
    if (output_size > compare_buffer_.size()) {
      compare_buffer_.resize(output_size);
    }
    size_t bytes_read = fread(&compare_buffer_[0],
                              1,
                              output_size,
                              output_file_);
    if (ferror(output_file_)) {
      std::cerr << "Error reading from " << output_file_type_ << " file '"
                << output_file_name_ << "': " << strerror(errno) << std::endl;
      return false;
    }
    if (bytes_read < output_size) {
      std::cerr << "Decoded target is longer than original target file"
                << std::endl;
      return false;
    }
    if (output.compare(0, output_size, &compare_buffer_[0], bytes_read) != 0) {
      std::cerr << "Original target file does not match decoded target"
                << std::endl;
      return false;
    }
  }
  return true;
}

bool VCDiffFileBasedCoder::Encode() {
  input_file_type_ = "target";
  input_file_name_ = FLAGS_target;
  output_file_type_ = "delta";
  output_file_name_ = FLAGS_delta;
  if (!OpenDictionary() || !OpenInputFile() || !OpenOutputFile()) {
    return false;
  }
  // Issue 6: Visual Studio STL produces a runtime exception
  // if &dictionary_[0] is attempted for an empty dictionary.
  if (dictionary_.empty()) {
    hashed_dictionary_.reset(new open_vcdiff::HashedDictionary("", 0));
  } else {
    hashed_dictionary_.reset(
        new open_vcdiff::HashedDictionary(&dictionary_[0],
                                          dictionary_.size()));
  }
  if (!hashed_dictionary_->Init()) {
    std::cerr << "Error initializing hashed dictionary" << std::endl;
    return false;
  }
  VCDiffFormatExtensionFlags format_flags = open_vcdiff::VCD_STANDARD_FORMAT;
  if (FLAGS_interleaved) {
    format_flags |= open_vcdiff::VCD_FORMAT_INTERLEAVED;
  }
  if (FLAGS_checksum) {
    format_flags |= open_vcdiff::VCD_FORMAT_CHECKSUM;
  }
  if (FLAGS_json) {
    format_flags |= open_vcdiff::VCD_FORMAT_JSON;
  }
  open_vcdiff::VCDiffStreamingEncoder encoder(hashed_dictionary_.get(),
                                              format_flags,
                                              FLAGS_target_matches);
  string output;
  size_t input_size = 0;
  size_t output_size = 0;
  {
    if (!encoder.StartEncoding(&output)) {
      std::cerr << "Error during encoder initialization" << std::endl;
      return false;
    }
  }
  do {
    size_t bytes_read = 0;
    if (!WriteOutput(output) || !ReadInput(&bytes_read)) {
      return false;
    }
    output_size += output.size();
    output.clear();
    if (bytes_read > 0) {
      input_size += bytes_read;
      if (!encoder.EncodeChunk(&input_buffer_[0], bytes_read, &output)) {
        std::cerr << "Error trying to encode data chunk of length "
                  << bytes_read << std::endl;
        return false;
      }
    }
  } while (!feof(input_file_));
  encoder.FinishEncoding(&output);
  if (!WriteOutput(output)) {
    return false;
  }
  output_size += output.size();
  output.clear();
  if (FLAGS_stats && (input_size > 0)) {
    std::cerr << "Original size: " << input_size
              << "\tCompressed size: " << output_size << " ("
              << ((static_cast<double>(output_size) / input_size) * 100)
              << "% of original)" << std::endl;
  }
  return true;
}

bool VCDiffFileBasedCoder::Decode() {
  input_file_type_ = "delta";
  input_file_name_ = FLAGS_delta;
  output_file_type_ = "target";
  output_file_name_ = FLAGS_target;
  if (!OpenDictionary() || !OpenInputFile() || !OpenOutputFile()) {
    return false;
  }

  open_vcdiff::VCDiffStreamingDecoder decoder;
  decoder.SetMaximumTargetFileSize(
      static_cast<size_t>(FLAGS_max_target_file_size));
  decoder.SetMaximumTargetWindowSize(
      static_cast<size_t>(FLAGS_max_target_window_size));
  decoder.SetAllowVcdTarget(FLAGS_allow_vcd_target);
  string output;
  size_t input_size = 0;
  size_t output_size = 0;
  // Issue 6: Visual Studio STL produces a runtime exception
  // if &dictionary_[0] is attempted for an empty dictionary.
  if (dictionary_.empty()) {
    decoder.StartDecoding("", 0);
  } else {
    decoder.StartDecoding(&dictionary_[0], dictionary_.size());
  }

  do {
    size_t bytes_read = 0;
    if (!ReadInput(&bytes_read)) {
      return false;
    }
    if (bytes_read > 0) {
      input_size += bytes_read;
      if (!decoder.DecodeChunk(&input_buffer_[0], bytes_read, &output)) {
        std::cerr << "Error trying to decode data chunk of length "
                  << bytes_read << std::endl;
        return false;
      }
    }
    if (!WriteOutput(output)) {
      return false;
    }
    output_size += output.size();
    output.clear();
  } while (!feof(input_file_));
  if (!decoder.FinishDecoding()) {
    std::cerr << "Decode error; '" << FLAGS_delta
              << " may not be a valid VCDIFF delta file" << std::endl;
    return false;
  }
  if (!WriteOutput(output)) {
    return false;
  }
  output_size += output.size();
  output.clear();
  if (FLAGS_stats && (output_size > 0)) {
    std::cerr << "Decompressed size: " << output_size
              << "\tCompressed size: " << input_size << " ("
              << ((static_cast<double>(input_size) / output_size) * 100)
              << "% of original)" << std::endl;
  }
  return true;
}

bool VCDiffFileBasedCoder::DecodeAndCompare() {
  input_file_type_ = "delta";
  input_file_name_ = FLAGS_delta;
  output_file_type_ = "target";
  output_file_name_ = FLAGS_target;
  if (!OpenDictionary() || !OpenInputFile() || !OpenOutputFileForCompare()) {
    return false;
  }

  open_vcdiff::VCDiffStreamingDecoder decoder;
  decoder.SetMaximumTargetFileSize(
      static_cast<size_t>(FLAGS_max_target_file_size));
  decoder.SetMaximumTargetWindowSize(
      static_cast<size_t>(FLAGS_max_target_window_size));
  decoder.SetAllowVcdTarget(FLAGS_allow_vcd_target);
  string output;
  size_t input_size = 0;
  size_t output_size = 0;
  // Issue 6: Visual Studio STL produces a runtime exception
  // if &dictionary_[0] is attempted for an empty dictionary.
  if (dictionary_.empty()) {
    decoder.StartDecoding("", 0);
  } else {
    decoder.StartDecoding(&dictionary_[0], dictionary_.size());
  }

  do {
    size_t bytes_read = 0;
    if (!ReadInput(&bytes_read)) {
      return false;
    }
    if (bytes_read > 0) {
      input_size += bytes_read;
      if (!decoder.DecodeChunk(&input_buffer_[0], bytes_read, &output)) {
        std::cerr << "Error trying to decode data chunk of length "
                  << bytes_read << std::endl;
        return false;
      }
    }
    if (!CompareOutput(output)) {
      return false;
    }
    output_size += output.size();
    output.clear();
  } while (!feof(input_file_));
  if (!decoder.FinishDecoding()) {
    std::cerr << "Decode error; '" << FLAGS_delta
              << " may not be a valid VCDIFF delta file" << std::endl;
    return false;
  }
  if (!CompareOutput(output)) {
    return false;
  }
  output_size += output.size();
  output.clear();
  if (fgetc(output_file_) != EOF) {
    std::cerr << "Decoded target is shorter than original target file"
              << std::endl;
    return false;
  }
  if (ferror(output_file_)) {
    std::cerr << "Error reading end-of-file indicator from target file"
              << std::endl;
    return false;
  }
  if (FLAGS_stats && (output_size > 0)) {
    std::cerr << "Decompressed size: " << output_size
              << "\tCompressed size: " << input_size << " ("
              << ((static_cast<double>(input_size) / output_size) * 100)
              << "% of original)" << std::endl;
  }
  return true;
}

}  // namespace open_vcdiff

int main(int argc, char** argv) {
  const char* const command_name = argv[0];
  google::SetUsageMessage(kUsageString);
  google::ParseCommandLineFlags(&argc, &argv, true);
  if (argc != 2) {
    std::cerr << command_name << ": Must specify exactly one command option"
              << std::endl;
    ShowUsageWithFlagsRestrict(command_name, "vcdiff");
    return 1;
  }
  const char* const command_option = argv[1];
  if (FLAGS_dictionary.empty()) {
    std::cerr << command_name << " " << command_option
              << ": Must specify --dictionary <file-name>" << std::endl;
    ShowUsageWithFlagsRestrict(command_name, "vcdiff");
    return 1;
  }
  if (!GetCommandLineFlagInfoOrDie("buffersize").is_default &&
       (FLAGS_buffersize == 0)) {
    std::cerr << command_name << ": Option --buffersize cannot be 0"
              << std::endl;
    ShowUsageWithFlagsRestrict(command_name, "vcdiff");
    return 1;
  }
  if ((strcmp(command_option, "encode") == 0) ||
      (strcmp(command_option, "delta") == 0)) {
    open_vcdiff::VCDiffFileBasedCoder coder;
    if (!coder.Encode()) {
      return 1;
    }
    // The destructor for VCDiffFileBasedCoder will clean up the open files
    // and allocated memory.
  } else if ((strcmp(command_option, "decode") == 0) ||
             (strcmp(command_option, "patch") == 0)) {
    open_vcdiff::VCDiffFileBasedCoder coder;
    if (!coder.Decode()) {
      return 1;
    }
  } else if ((strcmp(command_option, "test") == 0)) {
    // "vcdiff test" does not appear in the usage string, but can be
    // used for debugging.  It encodes, then decodes, then compares the result
    // with the original target. It expects the same arguments as
    // "vcdiff encode", with the additional requirement that the --target
    // and --delta file arguments must be specified, rather than using stdin
    // or stdout.  It produces a delta file just as for "vcdiff encode".
    if (FLAGS_target.empty() || FLAGS_delta.empty()) {
      std::cerr << command_name
                << " test: Must specify both --target <file-name>"
                   " and --delta <file-name>" << std::endl;
      return 1;
    }
    const string original_target(FLAGS_target);
    // Put coder into a separate scope.
    {
      open_vcdiff::VCDiffFileBasedCoder coder;
      if (!coder.Encode()) {
        return 1;
      }
    }
    {
      open_vcdiff::VCDiffFileBasedCoder coder;
      if (!coder.DecodeAndCompare()) {
        return 1;
      }
    }
  } else {
    std::cerr << command_name << ": Unrecognized command option "
              << command_option << std::endl;
    ShowUsageWithFlagsRestrict(command_name, "vcdiff");
    return 1;
  }
  return 0;
}