1 // Protocol Buffers - Google's data interchange format 2 // Copyright 2008 Google Inc. All rights reserved. 3 // https://developers.google.com/protocol-buffers/ 4 // 5 // Redistribution and use in source and binary forms, with or without 6 // modification, are permitted provided that the following conditions are 7 // met: 8 // 9 // * Redistributions of source code must retain the above copyright 10 // notice, this list of conditions and the following disclaimer. 11 // * Redistributions in binary form must reproduce the above 12 // copyright notice, this list of conditions and the following disclaimer 13 // in the documentation and/or other materials provided with the 14 // distribution. 15 // * Neither the name of Google Inc. nor the names of its 16 // contributors may be used to endorse or promote products derived from 17 // this software without specific prior written permission. 18 // 19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 // This file contains a program for running the test suite in a separate 32 // process. The other alternative is to run the suite in-process. See 33 // conformance.proto for pros/cons of these two options. 34 // 35 // This program will fork the process under test and communicate with it over 36 // its stdin/stdout: 37 // 38 // +--------+ pipe +----------+ 39 // | tester | <------> | testee | 40 // | | | | 41 // | C++ | | any lang | 42 // +--------+ +----------+ 43 // 44 // The tester contains all of the test cases and their expected output. 45 // The testee is a simple program written in the target language that reads 46 // each test case and attempts to produce acceptable output for it. 47 // 48 // Every test consists of a ConformanceRequest/ConformanceResponse 49 // request/reply pair. The protocol on the pipe is simply: 50 // 51 // 1. tester sends 4-byte length N (little endian) 52 // 2. tester sends N bytes representing a ConformanceRequest proto 53 // 3. testee sends 4-byte length M (little endian) 54 // 4. testee sends M bytes representing a ConformanceResponse proto 55 56 #include <algorithm> 57 #include <errno.h> 58 #include <fstream> 59 #include <sys/types.h> 60 #include <sys/wait.h> 61 #include <unistd.h> 62 #include <vector> 63 64 #include <google/protobuf/stubs/stringprintf.h> 65 66 #include "conformance.pb.h" 67 #include "conformance_test.h" 68 69 using conformance::ConformanceRequest; 70 using conformance::ConformanceResponse; 71 using google::protobuf::internal::scoped_array; 72 using google::protobuf::StringAppendF; 73 using std::string; 74 using std::vector; 75 76 #define STRINGIFY(x) #x 77 #define TOSTRING(x) STRINGIFY(x) 78 #define CHECK_SYSCALL(call) \ 79 if (call < 0) { \ 80 perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \ 81 exit(1); \ 82 } 83 84 // Test runner that spawns the process being tested and communicates with it 85 // over a pipe. 86 class ForkPipeRunner : public google::protobuf::ConformanceTestRunner { 87 public: 88 ForkPipeRunner(const std::string &executable) 89 : child_pid_(-1), executable_(executable) {} 90 91 virtual ~ForkPipeRunner() {} 92 93 void RunTest(const std::string& test_name, 94 const std::string& request, 95 std::string* response) { 96 if (child_pid_ < 0) { 97 SpawnTestProgram(); 98 } 99 100 current_test_name_ = test_name; 101 102 uint32_t len = request.size(); 103 CheckedWrite(write_fd_, &len, sizeof(uint32_t)); 104 CheckedWrite(write_fd_, request.c_str(), request.size()); 105 106 if (!TryRead(read_fd_, &len, sizeof(uint32_t))) { 107 // We failed to read from the child, assume a crash and try to reap. 108 GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_; 109 110 int status; 111 waitpid(child_pid_, &status, WEXITED); 112 113 string error_msg; 114 if (WIFEXITED(status)) { 115 StringAppendF(&error_msg, 116 "child exited, status=%d", WEXITSTATUS(status)); 117 } else if (WIFSIGNALED(status)) { 118 StringAppendF(&error_msg, 119 "child killed by signal %d", WTERMSIG(status)); 120 } 121 GOOGLE_LOG(INFO) << error_msg; 122 child_pid_ = -1; 123 124 conformance::ConformanceResponse response_obj; 125 response_obj.set_runtime_error(error_msg); 126 response_obj.SerializeToString(response); 127 return; 128 } 129 130 response->resize(len); 131 CheckedRead(read_fd_, (void*)response->c_str(), len); 132 } 133 134 private: 135 // TODO(haberman): make this work on Windows, instead of using these 136 // UNIX-specific APIs. 137 // 138 // There is a platform-agnostic API in 139 // src/google/protobuf/compiler/subprocess.h 140 // 141 // However that API only supports sending a single message to the subprocess. 142 // We really want to be able to send messages and receive responses one at a 143 // time: 144 // 145 // 1. Spawning a new process for each test would take way too long for thousands 146 // of tests and subprocesses like java that can take 100ms or more to start 147 // up. 148 // 149 // 2. Sending all the tests in one big message and receiving all results in one 150 // big message would take away our visibility about which test(s) caused a 151 // crash or other fatal error. It would also give us only a single failure 152 // instead of all of them. 153 void SpawnTestProgram() { 154 int toproc_pipe_fd[2]; 155 int fromproc_pipe_fd[2]; 156 if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) { 157 perror("pipe"); 158 exit(1); 159 } 160 161 pid_t pid = fork(); 162 if (pid < 0) { 163 perror("fork"); 164 exit(1); 165 } 166 167 if (pid) { 168 // Parent. 169 CHECK_SYSCALL(close(toproc_pipe_fd[0])); 170 CHECK_SYSCALL(close(fromproc_pipe_fd[1])); 171 write_fd_ = toproc_pipe_fd[1]; 172 read_fd_ = fromproc_pipe_fd[0]; 173 child_pid_ = pid; 174 } else { 175 // Child. 176 CHECK_SYSCALL(close(STDIN_FILENO)); 177 CHECK_SYSCALL(close(STDOUT_FILENO)); 178 CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO)); 179 CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO)); 180 181 CHECK_SYSCALL(close(toproc_pipe_fd[0])); 182 CHECK_SYSCALL(close(fromproc_pipe_fd[1])); 183 CHECK_SYSCALL(close(toproc_pipe_fd[1])); 184 CHECK_SYSCALL(close(fromproc_pipe_fd[0])); 185 186 scoped_array<char> executable(new char[executable_.size() + 1]); 187 memcpy(executable.get(), executable_.c_str(), executable_.size()); 188 executable[executable_.size()] = '\0'; 189 190 char *const argv[] = {executable.get(), NULL}; 191 CHECK_SYSCALL(execv(executable.get(), argv)); // Never returns. 192 } 193 } 194 195 void CheckedWrite(int fd, const void *buf, size_t len) { 196 if (write(fd, buf, len) != len) { 197 GOOGLE_LOG(FATAL) << current_test_name_ 198 << ": error writing to test program: " 199 << strerror(errno); 200 } 201 } 202 203 bool TryRead(int fd, void *buf, size_t len) { 204 size_t ofs = 0; 205 while (len > 0) { 206 ssize_t bytes_read = read(fd, (char*)buf + ofs, len); 207 208 if (bytes_read == 0) { 209 GOOGLE_LOG(ERROR) << current_test_name_ 210 << ": unexpected EOF from test program"; 211 return false; 212 } else if (bytes_read < 0) { 213 GOOGLE_LOG(ERROR) << current_test_name_ 214 << ": error reading from test program: " 215 << strerror(errno); 216 return false; 217 } 218 219 len -= bytes_read; 220 ofs += bytes_read; 221 } 222 223 return true; 224 } 225 226 void CheckedRead(int fd, void *buf, size_t len) { 227 if (!TryRead(fd, buf, len)) { 228 GOOGLE_LOG(FATAL) << current_test_name_ 229 << ": error reading from test program: " 230 << strerror(errno); 231 } 232 } 233 234 int write_fd_; 235 int read_fd_; 236 pid_t child_pid_; 237 std::string executable_; 238 std::string current_test_name_; 239 }; 240 241 void UsageError() { 242 fprintf(stderr, 243 "Usage: conformance-test-runner [options] <test-program>\n"); 244 fprintf(stderr, "\n"); 245 fprintf(stderr, "Options:\n"); 246 fprintf(stderr, 247 " --failure_list <filename> Use to specify list of tests\n"); 248 fprintf(stderr, 249 " that are expected to fail. File\n"); 250 fprintf(stderr, 251 " should contain one test name per\n"); 252 fprintf(stderr, 253 " line. Use '#' for comments.\n"); 254 exit(1); 255 } 256 257 void ParseFailureList(const char *filename, vector<string>* failure_list) { 258 std::ifstream infile(filename); 259 260 if (!infile.is_open()) { 261 fprintf(stderr, "Couldn't open failure list file: %s\n", filename); 262 exit(1); 263 } 264 265 for (string line; getline(infile, line);) { 266 // Remove whitespace. 267 line.erase(std::remove_if(line.begin(), line.end(), ::isspace), 268 line.end()); 269 270 // Remove comments. 271 line = line.substr(0, line.find("#")); 272 273 if (!line.empty()) { 274 failure_list->push_back(line); 275 } 276 } 277 } 278 279 int main(int argc, char *argv[]) { 280 char *program; 281 google::protobuf::ConformanceTestSuite suite; 282 283 vector<string> failure_list; 284 285 for (int arg = 1; arg < argc; ++arg) { 286 if (strcmp(argv[arg], "--failure_list") == 0) { 287 if (++arg == argc) UsageError(); 288 ParseFailureList(argv[arg], &failure_list); 289 } else if (strcmp(argv[arg], "--verbose") == 0) { 290 suite.SetVerbose(true); 291 } else if (argv[arg][0] == '-') { 292 fprintf(stderr, "Unknown option: %s\n", argv[arg]); 293 UsageError(); 294 } else { 295 if (arg != argc - 1) { 296 fprintf(stderr, "Too many arguments.\n"); 297 UsageError(); 298 } 299 program = argv[arg]; 300 } 301 } 302 303 suite.SetFailureList(failure_list); 304 ForkPipeRunner runner(program); 305 306 std::string output; 307 bool ok = suite.RunSuite(&runner, &output); 308 309 fwrite(output.c_str(), 1, output.size(), stderr); 310 311 return ok ? EXIT_SUCCESS : EXIT_FAILURE; 312 } 313
