1 /* Copyright (c) 2015, Google Inc. 2 * 3 * Permission to use, copy, modify, and/or distribute this software for any 4 * purpose with or without fee is hereby granted, provided that the above 5 * copyright notice and this permission notice appear in all copies. 6 * 7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ 14 15 #ifndef OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H 16 #define OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H 17 18 #include <openssl/base.h> 19 20 #include <stdint.h> 21 22 OPENSSL_MSVC_PRAGMA(warning(push)) 23 OPENSSL_MSVC_PRAGMA(warning(disable : 4702)) 24 25 #include <functional> 26 #include <map> 27 #include <memory> 28 #include <set> 29 #include <string> 30 #include <vector> 31 32 OPENSSL_MSVC_PRAGMA(warning(pop)) 33 34 // File-based test framework. 35 // 36 // This module provides a file-based test framework. The file format is based on 37 // that of OpenSSL upstream's evp_test and BoringSSL's aead_test. NIST CAVP test 38 // vector files are also supported. Each input file is a sequence of attributes, 39 // instructions and blank lines. 40 // 41 // Each attribute has the form: 42 // 43 // Name = Value 44 // 45 // Instructions are enclosed in square brackets and may appear without a value: 46 // 47 // [Name = Value] 48 // 49 // or 50 // 51 // [Name] 52 // 53 // Commas in instruction lines are treated as separate instructions. Thus this: 54 // 55 // [Name1,Name2] 56 // 57 // is the same as: 58 // 59 // [Name1] 60 // [Name2] 61 // 62 // Either '=' or ':' may be used to delimit the name from the value. Both the 63 // name and value have leading and trailing spaces stripped. 64 // 65 // Each file contains a number of instruction blocks and test cases. 66 // 67 // An instruction block is a sequence of instructions followed by a blank line. 68 // Instructions apply to all test cases following its appearance, until the next 69 // instruction block. Instructions are unordered. 70 // 71 // A test is a sequence of one or more attributes followed by a blank line. For 72 // tests that process multiple kinds of test cases, the first attribute is 73 // parsed out as the test's type and parameter. Otherwise, attributes are 74 // unordered. The first attribute is also included in the set of attributes, so 75 // tests which do not dispatch may ignore this mechanism. 76 // 77 // Additional blank lines and lines beginning with # are ignored. 78 // 79 // Functions in this module freely output to |stderr| on failure. Tests should 80 // also do so, and it is recommended they include the corresponding test's line 81 // number in any output. |PrintLine| does this automatically. 82 // 83 // Each attribute in a test and all instructions applying to it must be 84 // consumed. When a test completes, if any attributes or insturctions haven't 85 // been processed, the framework reports an error. 86 87 class FileTest; 88 typedef bool (*FileTestFunc)(FileTest *t, void *arg); 89 90 class FileTest { 91 public: 92 enum ReadResult { 93 kReadSuccess, 94 kReadEOF, 95 kReadError, 96 }; 97 98 class LineReader { 99 public: 100 virtual ~LineReader() {} 101 virtual ReadResult ReadLine(char *out, size_t len) = 0; 102 }; 103 104 struct Options { 105 // path is the path to the input file. 106 const char *path = nullptr; 107 // callback is called for each test. It should get the parameters from this 108 // object and signal any errors by returning false. 109 FileTestFunc callback = nullptr; 110 // arg is an opaque pointer that is passed to |callback|. 111 void *arg = nullptr; 112 // silent suppressed the "PASS" string that is otherwise printed after 113 // successful runs. 114 bool silent = false; 115 // comment_callback is called after each comment in the input is parsed. 116 std::function<void(const std::string&)> comment_callback; 117 // is_kas_test is true if a NIST KAS test is being parsed. These tests 118 // are inconsistent with the other NIST files to such a degree that they 119 // need their own boolean. 120 bool is_kas_test = false; 121 }; 122 123 explicit FileTest(std::unique_ptr<LineReader> reader, 124 std::function<void(const std::string &)> comment_callback, 125 bool is_kas_test); 126 ~FileTest(); 127 128 // ReadNext reads the next test from the file. It returns |kReadSuccess| if 129 // successfully reading a test and |kReadEOF| at the end of the file. On 130 // error or if the previous test had unconsumed attributes, it returns 131 // |kReadError|. 132 ReadResult ReadNext(); 133 134 // PrintLine is a variant of printf which prepends the line number and appends 135 // a trailing newline. 136 void PrintLine(const char *format, ...) OPENSSL_PRINTF_FORMAT_FUNC(2, 3); 137 138 unsigned start_line() const { return start_line_; } 139 140 // GetType returns the name of the first attribute of the current test. 141 const std::string &GetType(); 142 // GetParameter returns the value of the first attribute of the current test. 143 const std::string &GetParameter(); 144 145 // HasAttribute returns true if the current test has an attribute named |key|. 146 bool HasAttribute(const std::string &key); 147 148 // GetAttribute looks up the attribute with key |key|. It sets |*out_value| to 149 // the value and returns true if it exists and returns false with an error to 150 // |stderr| otherwise. 151 bool GetAttribute(std::string *out_value, const std::string &key); 152 153 // GetAttributeOrDie looks up the attribute with key |key| and aborts if it is 154 // missing. It should only be used after a |HasAttribute| call. 155 const std::string &GetAttributeOrDie(const std::string &key); 156 157 // IgnoreAttribute marks the attribute with key |key| as used. 158 void IgnoreAttribute(const std::string &key) { HasAttribute(key); } 159 160 // GetBytes looks up the attribute with key |key| and decodes it as a byte 161 // string. On success, it writes the result to |*out| and returns 162 // true. Otherwise it returns false with an error to |stderr|. The value may 163 // be either a hexadecimal string or a quoted ASCII string. It returns true on 164 // success and returns false with an error to |stderr| on failure. 165 bool GetBytes(std::vector<uint8_t> *out, const std::string &key); 166 167 // ExpectBytesEqual returns true if |expected| and |actual| are equal. 168 // Otherwise, it returns false and prints a message to |stderr|. 169 bool ExpectBytesEqual(const uint8_t *expected, size_t expected_len, 170 const uint8_t *actual, size_t actual_len); 171 172 // AtNewInstructionBlock returns true if the current test was immediately 173 // preceded by an instruction block. 174 bool IsAtNewInstructionBlock() const; 175 176 // HasInstruction returns true if the current test has an instruction. 177 bool HasInstruction(const std::string &key); 178 179 // IgnoreInstruction marks the instruction with key |key| as used. 180 void IgnoreInstruction(const std::string &key) { HasInstruction(key); } 181 182 // GetInstruction looks up the instruction with key |key|. It sets 183 // |*out_value| to the value (empty string if the instruction has no value) 184 // and returns true if it exists and returns false with an error to |stderr| 185 // otherwise. 186 bool GetInstruction(std::string *out_value, const std::string &key); 187 188 // GetInstructionOrDie looks up the instruction with key |key| and aborts if 189 // it is missing. It should only be used after a |HasInstruction| call. 190 const std::string &GetInstructionOrDie(const std::string &key); 191 192 // GetInstructionBytes behaves like GetBytes, but looks up the corresponding 193 // instruction. 194 bool GetInstructionBytes(std::vector<uint8_t> *out, const std::string &key); 195 196 // CurrentTestToString returns the file content parsed for the current test. 197 // If the current test was preceded by an instruction block, the return test 198 // case is preceded by the instruction block and a single blank line. All 199 // other blank or comment lines are omitted. 200 const std::string &CurrentTestToString() const; 201 202 // InjectInstruction adds a key value pair to the most recently parsed set of 203 // instructions. 204 void InjectInstruction(const std::string &key, const std::string &value); 205 206 // SkipCurrent passes the current test case. Unused attributes are ignored. 207 void SkipCurrent(); 208 209 private: 210 void ClearTest(); 211 void ClearInstructions(); 212 void OnKeyUsed(const std::string &key); 213 void OnInstructionUsed(const std::string &key); 214 bool ConvertToBytes(std::vector<uint8_t> *out, const std::string &value); 215 216 std::unique_ptr<LineReader> reader_; 217 // line_ is the number of lines read. 218 unsigned line_ = 0; 219 220 // start_line_ is the line number of the first attribute of the test. 221 unsigned start_line_ = 0; 222 // type_ is the name of the first attribute of the test. 223 std::string type_; 224 // parameter_ is the value of the first attribute. 225 std::string parameter_; 226 // attributes_ contains all attributes in the test, including the first. 227 std::map<std::string, std::string> attributes_; 228 // instructions_ contains all instructions in scope for the test. 229 std::map<std::string, std::string> instructions_; 230 231 // unused_attributes_ is the set of attributes that have not been queried. 232 std::set<std::string> unused_attributes_; 233 234 // unused_instructions_ is the set of instructions that have not been queried. 235 std::set<std::string> unused_instructions_; 236 237 std::string current_test_; 238 239 bool is_at_new_instruction_block_ = false; 240 bool seen_non_comment_ = false; 241 bool is_kas_test_ = false; 242 243 // comment_callback_, if set, is a callback function that is called with the 244 // contents of each comment as they are parsed. 245 std::function<void(const std::string&)> comment_callback_; 246 247 FileTest(const FileTest &) = delete; 248 FileTest &operator=(const FileTest &) = delete; 249 }; 250 251 // FileTestMain runs a file-based test out of |path| and returns an exit code 252 // suitable to return out of |main|. |run_test| should return true on pass and 253 // false on failure. FileTestMain also implements common handling of the 'Error' 254 // attribute. A test with that attribute is expected to fail. The value of the 255 // attribute is the reason string of the expected OpenSSL error code. 256 // 257 // Tests are guaranteed to run serially and may affect global state if need be. 258 // It is legal to use "tests" which, for example, import a private key into a 259 // list of keys. This may be used to initialize a shared set of keys for many 260 // tests. However, if one test fails, the framework will continue to run 261 // subsequent tests. 262 int FileTestMain(FileTestFunc run_test, void *arg, const char *path); 263 264 // FileTestMain accepts a larger number of options via a struct. 265 int FileTestMain(const FileTest::Options &opts); 266 267 // FileTestGTest behaves like FileTestMain, but for GTest. |path| must be the 268 // name of a test file embedded in the test binary. 269 void FileTestGTest(const char *path, std::function<void(FileTest *)> run_test); 270 271 #endif // OPENSSL_HEADER_CRYPTO_TEST_FILE_TEST_H 272
