1 /* 2 frameTest - test tool for lz4frame 3 Copyright (C) Yann Collet 2014-2016 4 5 GPL v2 License 6 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 2 of the License, or 10 (at your option) any later version. 11 12 This program is distributed in the hope that it will be useful, 13 but WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program; if not, write to the Free Software Foundation, Inc., 19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 21 You can contact the author at : 22 - LZ4 homepage : http://www.lz4.org 23 - LZ4 source repository : https://github.com/lz4/lz4 24 */ 25 26 /*-************************************ 27 * Compiler specific 28 **************************************/ 29 #ifdef _MSC_VER /* Visual Studio */ 30 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 31 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ 32 #endif 33 34 35 /*-************************************ 36 * Includes 37 **************************************/ 38 #include "util.h" /* U32 */ 39 #include <stdlib.h> /* malloc, free */ 40 #include <stdio.h> /* fprintf */ 41 #include <string.h> /* strcmp */ 42 #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ 43 #include "lz4frame_static.h" 44 #include "lz4.h" /* LZ4_VERSION_STRING */ 45 #define XXH_STATIC_LINKING_ONLY 46 #include "xxhash.h" /* XXH64 */ 47 48 49 /* unoptimized version; solves endianess & alignment issues */ 50 static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) 51 { 52 BYTE* dstPtr = (BYTE*)dstVoidPtr; 53 dstPtr[0] = (BYTE)value32; 54 dstPtr[1] = (BYTE)(value32 >> 8); 55 dstPtr[2] = (BYTE)(value32 >> 16); 56 dstPtr[3] = (BYTE)(value32 >> 24); 57 } 58 59 60 /*-************************************ 61 * Constants 62 **************************************/ 63 #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U 64 65 #define KB *(1U<<10) 66 #define MB *(1U<<20) 67 #define GB *(1U<<30) 68 69 static const U32 nbTestsDefault = 256 KB; 70 #define COMPRESSIBLE_NOISE_LENGTH (2 MB) 71 #define FUZ_COMPRESSIBILITY_DEFAULT 50 72 static const U32 prime1 = 2654435761U; 73 static const U32 prime2 = 2246822519U; 74 75 76 77 /*-************************************ 78 * Macros 79 **************************************/ 80 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 81 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } 82 #define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ 83 if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \ 84 { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \ 85 if (displayLevel>=4) fflush(stdout); } } 86 static const clock_t refreshRate = CLOCKS_PER_SEC / 6; 87 static clock_t g_clockTime = 0; 88 89 90 /*-*************************************** 91 * Local Parameters 92 *****************************************/ 93 static U32 no_prompt = 0; 94 static U32 displayLevel = 2; 95 static U32 use_pause = 0; 96 97 98 /*-******************************************************* 99 * Fuzzer functions 100 *********************************************************/ 101 #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) 102 #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) 103 104 static clock_t FUZ_GetClockSpan(clock_t clockStart) 105 { 106 return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ 107 } 108 109 110 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) 111 unsigned int FUZ_rand(unsigned int* src) 112 { 113 U32 rand32 = *src; 114 rand32 *= prime1; 115 rand32 += prime2; 116 rand32 = FUZ_rotl32(rand32, 13); 117 *src = rand32; 118 return rand32 >> 5; 119 } 120 121 122 #define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) 123 #define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) 124 static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) 125 { 126 BYTE* BBuffer = (BYTE*)buffer; 127 size_t pos = 0; 128 U32 P32 = (U32)(32768 * proba); 129 130 /* First Byte */ 131 BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); 132 133 while (pos < bufferSize) { 134 /* Select : Literal (noise) or copy (within 64K) */ 135 if (FUZ_RAND15BITS < P32) { 136 /* Copy (within 64K) */ 137 size_t const lengthRand = FUZ_RANDLENGTH + 4; 138 size_t const length = MIN(lengthRand, bufferSize - pos); 139 size_t const end = pos + length; 140 size_t const offsetRand = FUZ_RAND15BITS + 1; 141 size_t const offset = MIN(offsetRand, pos); 142 size_t match = pos - offset; 143 while (pos < end) BBuffer[pos++] = BBuffer[match++]; 144 } else { 145 /* Literal (noise) */ 146 size_t const lengthRand = FUZ_RANDLENGTH + 4; 147 size_t const length = MIN(lengthRand, bufferSize - pos); 148 size_t const end = pos + length; 149 while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); 150 } 151 } 152 } 153 154 155 static unsigned FUZ_highbit(U32 v32) 156 { 157 unsigned nbBits = 0; 158 if (v32==0) return 0; 159 while (v32) v32 >>= 1, nbBits ++; 160 return nbBits; 161 } 162 163 164 /*-******************************************************* 165 * Tests 166 *********************************************************/ 167 int basicTests(U32 seed, double compressibility) 168 { 169 int testResult = 0; 170 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); 171 size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL); 172 void* const compressedBuffer = malloc(cBuffSize); 173 void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); 174 U32 randState = seed; 175 size_t cSize, testSize; 176 LZ4F_decompressionContext_t dCtx = NULL; 177 LZ4F_compressionContext_t cctx = NULL; 178 U64 crcOrig; 179 180 LZ4F_preferences_t prefs; 181 memset(&prefs, 0, sizeof(prefs)); 182 if (!CNBuffer || !compressedBuffer || !decodedBuffer) { 183 DISPLAY("allocation error, not enough memory to start fuzzer tests \n"); 184 goto _output_error; 185 } 186 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); 187 crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 188 189 /* LZ4F_compressBound() : special case : srcSize == 0 */ 190 DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); 191 { size_t const cBound = LZ4F_compressBound(0, NULL); 192 if (cBound < 64 KB) goto _output_error; 193 DISPLAYLEVEL(3, " %u \n", (U32)cBound); 194 } 195 196 /* Special case : null-content frame */ 197 testSize = 0; 198 DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); 199 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); 200 if (LZ4F_isError(cSize)) goto _output_error; 201 DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize); 202 203 DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); 204 { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 205 if (LZ4F_isError(errorCode)) goto _output_error; } 206 207 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); 208 { size_t avail_in = cSize; 209 LZ4F_frameInfo_t frame_info; 210 LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in); 211 if (LZ4F_isError(errorCode)) goto _output_error; 212 } 213 214 DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); 215 { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx); 216 if (LZ4F_isError(errorCode)) goto _output_error; } 217 dCtx = NULL; 218 219 /* test one-pass frame compression */ 220 testSize = COMPRESSIBLE_NOISE_LENGTH; 221 DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n"); 222 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); 223 if (LZ4F_isError(cSize)) goto _output_error; 224 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 225 226 DISPLAYLEVEL(3, "Decompression test : \n"); 227 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 228 size_t compressedBufferSize = cSize; 229 BYTE* ip = (BYTE*)compressedBuffer; 230 BYTE* const iend = (BYTE*)compressedBuffer + cSize; 231 232 LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 233 if (LZ4F_isError(errorCode)) goto _output_error; 234 235 DISPLAYLEVEL(3, "Single Pass decompression : \n"); 236 { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); 237 if (LZ4F_isError(decompressError)) goto _output_error; } 238 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 239 if (crcDest != crcOrig) goto _output_error; } 240 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize); 241 242 DISPLAYLEVEL(3, "Reusing decompression context \n"); 243 { size_t const missingBytes = 4; 244 size_t iSize = compressedBufferSize - missingBytes; 245 const BYTE* cBuff = (const BYTE*) compressedBuffer; 246 BYTE* const ostart = (BYTE*)decodedBuffer; 247 BYTE* op = ostart; 248 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 249 size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH; 250 DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes); 251 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); 252 if (LZ4F_isError(decResult)) goto _output_error; 253 if (decResult != missingBytes) { 254 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult); 255 goto _output_error; 256 } 257 DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult); 258 cBuff += iSize; 259 iSize = decResult; 260 op += oSize; 261 oSize = (size_t)(oend-op); 262 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); 263 if (decResult != 0) goto _output_error; /* should finish now */ 264 op += oSize; 265 if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; } 266 { U64 const crcDest = XXH64(decodedBuffer, op-ostart, 1); 267 if (crcDest != crcOrig) goto _output_error; 268 } } 269 270 { size_t oSize = 0; 271 size_t iSize = 0; 272 LZ4F_frameInfo_t fi; 273 274 DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); 275 errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); 276 if (LZ4F_isError(errorCode)) goto _output_error; 277 DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); 278 279 DISPLAYLEVEL(3, "get FrameInfo on null input : "); 280 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); 281 if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; 282 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); 283 284 DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); 285 iSize = 6; 286 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); 287 if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; 288 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); 289 ip += iSize; 290 291 DISPLAYLEVEL(3, "get FrameInfo on enough input : "); 292 iSize = 15 - iSize; 293 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); 294 if (LZ4F_isError(errorCode)) goto _output_error; 295 DISPLAYLEVEL(3, " correctly decoded \n"); 296 ip += iSize; 297 } 298 299 DISPLAYLEVEL(3, "Byte after byte : \n"); 300 { BYTE* const ostart = (BYTE*)decodedBuffer; 301 BYTE* op = ostart; 302 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 303 while (ip < iend) { 304 size_t oSize = oend-op; 305 size_t iSize = 1; 306 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 307 if (LZ4F_isError(errorCode)) goto _output_error; 308 op += oSize; 309 ip += iSize; 310 } 311 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 312 if (crcDest != crcOrig) goto _output_error; } 313 DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH); 314 } 315 316 errorCode = LZ4F_freeDecompressionContext(dCtx); 317 if (LZ4F_isError(errorCode)) goto _output_error; 318 dCtx = NULL; 319 } 320 321 DISPLAYLEVEL(3, "Using 64 KB block : \n"); 322 prefs.frameInfo.blockSizeID = LZ4F_max64KB; 323 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 324 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 325 if (LZ4F_isError(cSize)) goto _output_error; 326 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 327 328 DISPLAYLEVEL(3, "without checksum : \n"); 329 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 330 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 331 if (LZ4F_isError(cSize)) goto _output_error; 332 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 333 334 DISPLAYLEVEL(3, "Using 256 KB block : \n"); 335 prefs.frameInfo.blockSizeID = LZ4F_max256KB; 336 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 337 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 338 if (LZ4F_isError(cSize)) goto _output_error; 339 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 340 341 DISPLAYLEVEL(3, "Decompression test : \n"); 342 { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 343 unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize); 344 BYTE* const ostart = (BYTE*)decodedBuffer; 345 BYTE* op = ostart; 346 BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH; 347 const BYTE* ip = (const BYTE*)compressedBuffer; 348 const BYTE* const iend = (const BYTE*)compressedBuffer + cSize; 349 350 { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 351 if (LZ4F_isError(createError)) goto _output_error; } 352 353 DISPLAYLEVEL(3, "random segment sizes : \n"); 354 while (ip < iend) { 355 unsigned const nbBits = FUZ_rand(&randState) % maxBits; 356 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 357 size_t oSize = oend-op; 358 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 359 { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 360 if (LZ4F_isError(decompressError)) goto _output_error; } 361 op += oSize; 362 ip += iSize; 363 } 364 { size_t const decodedSize = (size_t)(op - ostart); 365 U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); 366 if (crcDest != crcOrig) goto _output_error; 367 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); 368 } 369 370 { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx); 371 if (LZ4F_isError(freeError)) goto _output_error; } 372 dCtx = NULL; 373 } 374 375 DISPLAYLEVEL(3, "without checksum : \n"); 376 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 377 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 378 if (LZ4F_isError(cSize)) goto _output_error; 379 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 380 381 DISPLAYLEVEL(3, "Using 1 MB block : \n"); 382 prefs.frameInfo.blockSizeID = LZ4F_max1MB; 383 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 384 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 385 if (LZ4F_isError(cSize)) goto _output_error; 386 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 387 388 DISPLAYLEVEL(3, "without checksum : \n"); 389 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 390 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); 391 if (LZ4F_isError(cSize)) goto _output_error; 392 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 393 394 DISPLAYLEVEL(3, "Using 4 MB block : \n"); 395 prefs.frameInfo.blockSizeID = LZ4F_max4MB; 396 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 397 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); 398 DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) 399 cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); 400 if (LZ4F_isError(cSize)) goto _output_error; 401 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 402 } 403 404 DISPLAYLEVEL(3, "without checksum : \n"); 405 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 406 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); 407 DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity) 408 cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs); 409 if (LZ4F_isError(cSize)) goto _output_error; 410 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 411 } 412 413 { size_t errorCode; 414 BYTE* const ostart = (BYTE*)compressedBuffer; 415 BYTE* op = ostart; 416 errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); 417 if (LZ4F_isError(errorCode)) goto _output_error; 418 419 DISPLAYLEVEL(3, "compress without frameSize : \n"); 420 memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); 421 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); 422 if (LZ4F_isError(errorCode)) goto _output_error; 423 op += errorCode; 424 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); 425 if (LZ4F_isError(errorCode)) goto _output_error; 426 op += errorCode; 427 errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); 428 if (LZ4F_isError(errorCode)) goto _output_error; 429 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); 430 431 DISPLAYLEVEL(3, "compress with frameSize : \n"); 432 prefs.frameInfo.contentSize = testSize; 433 op = ostart; 434 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); 435 if (LZ4F_isError(errorCode)) goto _output_error; 436 op += errorCode; 437 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); 438 if (LZ4F_isError(errorCode)) goto _output_error; 439 op += errorCode; 440 errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL); 441 if (LZ4F_isError(errorCode)) goto _output_error; 442 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); 443 444 DISPLAYLEVEL(3, "compress with wrong frameSize : \n"); 445 prefs.frameInfo.contentSize = testSize+1; 446 op = ostart; 447 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs); 448 if (LZ4F_isError(errorCode)) goto _output_error; 449 op += errorCode; 450 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); 451 if (LZ4F_isError(errorCode)) goto _output_error; 452 op += errorCode; 453 errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL); 454 if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); } 455 else 456 goto _output_error; 457 458 errorCode = LZ4F_freeCompressionContext(cctx); 459 if (LZ4F_isError(errorCode)) goto _output_error; 460 cctx = NULL; 461 } 462 463 DISPLAYLEVEL(3, "Skippable frame test : \n"); 464 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 465 unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); 466 BYTE* op = (BYTE*)decodedBuffer; 467 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 468 BYTE* ip = (BYTE*)compressedBuffer; 469 BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; 470 471 LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 472 if (LZ4F_isError(errorCode)) goto _output_error; 473 474 /* generate skippable frame */ 475 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START); 476 FUZ_writeLE32(ip+4, (U32)cSize); 477 478 DISPLAYLEVEL(3, "random segment sizes : \n"); 479 while (ip < iend) { 480 unsigned nbBits = FUZ_rand(&randState) % maxBits; 481 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 482 size_t oSize = oend-op; 483 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 484 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 485 if (LZ4F_isError(errorCode)) goto _output_error; 486 op += oSize; 487 ip += iSize; 488 } 489 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); 490 491 /* generate zero-size skippable frame */ 492 DISPLAYLEVEL(3, "zero-size skippable frame\n"); 493 ip = (BYTE*)compressedBuffer; 494 op = (BYTE*)decodedBuffer; 495 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1); 496 FUZ_writeLE32(ip+4, 0); 497 iend = ip+8; 498 499 while (ip < iend) { 500 unsigned nbBits = FUZ_rand(&randState) % maxBits; 501 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 502 size_t oSize = oend-op; 503 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 504 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 505 if (LZ4F_isError(errorCode)) goto _output_error; 506 op += oSize; 507 ip += iSize; 508 } 509 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); 510 511 DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); 512 ip = (BYTE*)compressedBuffer; 513 op = (BYTE*)decodedBuffer; 514 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2); 515 FUZ_writeLE32(ip+4, 10); 516 iend = ip+18; 517 while (ip < iend) { 518 size_t iSize = 10; 519 size_t oSize = 10; 520 if (iSize > (size_t)(iend-ip)) iSize = iend-ip; 521 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); 522 if (LZ4F_isError(errorCode)) goto _output_error; 523 op += oSize; 524 ip += iSize; 525 } 526 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); 527 } 528 529 DISPLAY("Basic tests completed \n"); 530 _end: 531 free(CNBuffer); 532 free(compressedBuffer); 533 free(decodedBuffer); 534 LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; 535 LZ4F_freeCompressionContext(cctx); cctx = NULL; 536 return testResult; 537 538 _output_error: 539 testResult = 1; 540 DISPLAY("Error detected ! \n"); 541 goto _end; 542 } 543 544 545 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) 546 { 547 int p=0; 548 const BYTE* b1=(const BYTE*)buff1; 549 const BYTE* b2=(const BYTE*)buff2; 550 if (nonContiguous) { 551 DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); 552 return; 553 } 554 while (b1[p]==b2[p]) p++; 555 DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]); 556 } 557 558 559 int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) 560 { 561 unsigned testResult = 0; 562 unsigned testNb = 0; 563 size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ 564 void* srcBuffer = NULL; 565 size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL); 566 void* compressedBuffer = NULL; 567 void* decodedBuffer = NULL; 568 U32 coreRand = seed; 569 LZ4F_decompressionContext_t dCtx = NULL; 570 LZ4F_compressionContext_t cCtx = NULL; 571 size_t result; 572 clock_t const startClock = clock(); 573 clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; 574 # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ 575 DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } 576 577 /* Create buffers */ 578 result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 579 CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); 580 result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); 581 CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); 582 srcBuffer = malloc(srcDataLength); 583 CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); 584 compressedBuffer = malloc(compressedBufferSize); 585 CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); 586 decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ 587 CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); 588 FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); 589 590 /* jump to requested testNb */ 591 for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */ 592 593 /* main fuzzer test loop */ 594 for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { 595 U32 randState = coreRand ^ prime1; 596 unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1; 597 size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1; 598 size_t const srcStartId = FUZ_rand(&randState) % (srcDataLength - srcSize); 599 const BYTE* const srcStart = (const BYTE*)srcBuffer + srcStartId; 600 unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1; 601 U64 const crcOrig = XXH64(srcStart, srcSize, 1); 602 LZ4F_preferences_t prefs; 603 const LZ4F_preferences_t* prefsPtr = &prefs; 604 size_t cSize; 605 606 (void)FUZ_rand(&coreRand); /* update seed */ 607 memset(&prefs, 0, sizeof(prefs)); 608 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1); 609 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3)); 610 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1); 611 prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; 612 prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2; 613 prefs.compressionLevel = FUZ_rand(&randState) % 5; 614 if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; 615 616 DISPLAYUPDATE(2, "\r%5u ", testNb); 617 618 if ((FUZ_rand(&randState) & 0xFFF) == 0) { 619 /* create a skippable frame (rare case) */ 620 BYTE* op = (BYTE*)compressedBuffer; 621 FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15)); 622 FUZ_writeLE32(op+4, (U32)srcSize); 623 cSize = srcSize+8; 624 } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */ 625 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr); 626 CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); 627 } else { /* multi-segments compression */ 628 const BYTE* ip = srcStart; 629 const BYTE* const iend = srcStart + srcSize; 630 BYTE* op = (BYTE*)compressedBuffer; 631 BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */ 632 unsigned const maxBits = FUZ_highbit((U32)srcSize); 633 LZ4F_compressOptions_t cOptions; 634 memset(&cOptions, 0, sizeof(cOptions)); 635 result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr); 636 CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result); 637 op += result; 638 while (ip < iend) { 639 unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits; 640 size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; 641 size_t const iSize = MIN(sampleMax, (size_t)(iend-ip)); 642 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); 643 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); 644 645 result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); 646 CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); 647 op += result; 648 ip += iSize; 649 650 { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1); 651 if (forceFlush) { 652 result = LZ4F_flush(cCtx, op, oend-op, &cOptions); 653 CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result); 654 op += result; 655 } } 656 } 657 CHECK(op>=oend, "LZ4F_compressFrameBound overflow"); 658 result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions); 659 CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result)); 660 op += result; 661 cSize = op-(BYTE*)compressedBuffer; 662 DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); 663 } 664 665 /* multi-segments decompression */ 666 { const BYTE* ip = (const BYTE*)compressedBuffer; 667 const BYTE* const iend = ip + cSize; 668 BYTE* op = (BYTE*)decodedBuffer; 669 BYTE* const oend = op + srcDataLength; 670 unsigned const suggestedBits = FUZ_highbit((U32)cSize); 671 unsigned const maxBits = MAX(3, suggestedBits); 672 unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ 673 size_t totalOut = 0; 674 XXH64_state_t xxh64; 675 XXH64_reset(&xxh64, 1); 676 while (ip < iend) { 677 unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; 678 unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; 679 size_t const iSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; 680 size_t iSize = MIN(iSizeMax, (size_t)(iend-ip)); 681 size_t const oSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; 682 size_t oSize = MIN(oSizeMax, (size_t)(oend-op)); 683 LZ4F_decompressOptions_t dOptions; 684 memset(&dOptions, 0, sizeof(dOptions)); 685 dOptions.stableDst = FUZ_rand(&randState) & 1; 686 if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */ 687 result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); 688 if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); 689 CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName(result)); 690 XXH64_update(&xxh64, op, (U32)oSize); 691 totalOut += oSize; 692 op += oSize; 693 ip += iSize; 694 op += nonContiguousDst; 695 if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ 696 } 697 CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); 698 if (totalOut) { /* otherwise, it's a skippable frame */ 699 U64 const crcDecoded = XXH64_digest(&xxh64); 700 if (crcDecoded != crcOrig) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); 701 CHECK(crcDecoded != crcOrig, "Decompression corruption"); 702 } 703 } 704 } 705 706 DISPLAYLEVEL(2, "\rAll tests completed \n"); 707 708 _end: 709 LZ4F_freeDecompressionContext(dCtx); 710 LZ4F_freeCompressionContext(cCtx); 711 free(srcBuffer); 712 free(compressedBuffer); 713 free(decodedBuffer); 714 715 if (use_pause) { 716 DISPLAY("press enter to finish \n"); 717 (void)getchar(); 718 } 719 return testResult; 720 721 _output_error: 722 testResult = 1; 723 goto _end; 724 } 725 726 727 int FUZ_usage(const char* programName) 728 { 729 DISPLAY( "Usage :\n"); 730 DISPLAY( " %s [args]\n", programName); 731 DISPLAY( "\n"); 732 DISPLAY( "Arguments :\n"); 733 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); 734 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); 735 DISPLAY( " -s# : Select seed (default:prompt user)\n"); 736 DISPLAY( " -t# : Select starting test number (default:0)\n"); 737 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); 738 DISPLAY( " -v : verbose\n"); 739 DISPLAY( " -h : display help and exit\n"); 740 return 0; 741 } 742 743 744 int main(int argc, const char** argv) 745 { 746 U32 seed=0; 747 int seedset=0; 748 int argNb; 749 int nbTests = nbTestsDefault; 750 int testNb = 0; 751 int proba = FUZ_COMPRESSIBILITY_DEFAULT; 752 int result=0; 753 U32 duration=0; 754 const char* const programName = argv[0]; 755 756 /* Check command line */ 757 for (argNb=1; argNb<argc; argNb++) { 758 const char* argument = argv[argNb]; 759 760 if(!argument) continue; /* Protection if argument empty */ 761 762 /* Decode command (note : aggregated short commands are allowed) */ 763 if (argument[0]=='-') { 764 if (!strcmp(argument, "--no-prompt")) { 765 no_prompt=1; 766 seedset=1; 767 displayLevel=1; 768 continue; 769 } 770 argument++; 771 772 while (*argument!=0) { 773 switch(*argument) 774 { 775 case 'h': 776 return FUZ_usage(programName); 777 case 'v': 778 argument++; 779 displayLevel++; 780 break; 781 case 'q': 782 argument++; 783 displayLevel--; 784 break; 785 case 'p': /* pause at the end */ 786 argument++; 787 use_pause = 1; 788 break; 789 790 case 'i': 791 argument++; 792 nbTests=0; duration=0; 793 while ((*argument>='0') && (*argument<='9')) { 794 nbTests *= 10; 795 nbTests += *argument - '0'; 796 argument++; 797 } 798 break; 799 800 case 'T': 801 argument++; 802 nbTests = 0; duration = 0; 803 for (;;) { 804 switch(*argument) 805 { 806 case 'm': duration *= 60; argument++; continue; 807 case 's': 808 case 'n': argument++; continue; 809 case '0': 810 case '1': 811 case '2': 812 case '3': 813 case '4': 814 case '5': 815 case '6': 816 case '7': 817 case '8': 818 case '9': duration *= 10; duration += *argument++ - '0'; continue; 819 } 820 break; 821 } 822 break; 823 824 case 's': 825 argument++; 826 seed=0; 827 seedset=1; 828 while ((*argument>='0') && (*argument<='9')) { 829 seed *= 10; 830 seed += *argument - '0'; 831 argument++; 832 } 833 break; 834 case 't': 835 argument++; 836 testNb=0; 837 while ((*argument>='0') && (*argument<='9')) { 838 testNb *= 10; 839 testNb += *argument - '0'; 840 argument++; 841 } 842 break; 843 case 'P': /* compressibility % */ 844 argument++; 845 proba=0; 846 while ((*argument>='0') && (*argument<='9')) { 847 proba *= 10; 848 proba += *argument - '0'; 849 argument++; 850 } 851 if (proba<0) proba=0; 852 if (proba>100) proba=100; 853 break; 854 default: 855 ; 856 return FUZ_usage(programName); 857 } 858 } 859 } 860 } 861 862 /* Get Seed */ 863 DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); 864 865 if (!seedset) { 866 time_t const t = time(NULL); 867 U32 const h = XXH32(&t, sizeof(t), 1); 868 seed = h % 10000; 869 } 870 DISPLAY("Seed = %u\n", seed); 871 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); 872 873 if (nbTests<=0) nbTests=1; 874 875 if (testNb==0) result = basicTests(seed, ((double)proba) / 100); 876 if (result) return 1; 877 return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); 878 } 879
