1 /* 2 LZ4io.c - LZ4 File/Stream Interface 3 Copyright (C) Yann Collet 2011-2016 4 5 GPL v2 License 6 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 2 of the License, or 10 (at your option) any later version. 11 12 This program is distributed in the hope that it will be useful, 13 but WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program; if not, write to the Free Software Foundation, Inc., 19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 21 You can contact the author at : 22 - LZ4 source repository : https://github.com/lz4/lz4 23 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 24 */ 25 /* 26 Note : this is stand-alone program. 27 It is not part of LZ4 compression library, it is a user code of the LZ4 library. 28 - The license of LZ4 library is BSD. 29 - The license of xxHash library is BSD. 30 - The license of this source file is GPLv2. 31 */ 32 33 34 /*-************************************ 35 * Compiler options 36 **************************************/ 37 #ifdef _MSC_VER /* Visual Studio */ 38 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 39 #endif 40 #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) 41 # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ 42 #endif 43 44 45 /***************************** 46 * Includes 47 *****************************/ 48 #include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ 49 #include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ 50 #include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ 51 #include <stdlib.h> /* malloc, free */ 52 #include <string.h> /* strerror, strcmp, strlen */ 53 #include <time.h> /* clock */ 54 #include <sys/types.h> /* stat64 */ 55 #include <sys/stat.h> /* stat64 */ 56 #include "lz4io.h" 57 #include "lz4.h" /* still required for legacy format */ 58 #include "lz4hc.h" /* still required for legacy format */ 59 #include "lz4frame.h" 60 61 62 /* ************************************** 63 * Compiler Options 64 ****************************************/ 65 #if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ 66 # define fseek _fseeki64 67 #endif 68 #if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ 69 # define fseek fseeko 70 #endif 71 72 73 /***************************** 74 * Constants 75 *****************************/ 76 #define KB *(1 <<10) 77 #define MB *(1 <<20) 78 #define GB *(1U<<30) 79 80 #define _1BIT 0x01 81 #define _2BITS 0x03 82 #define _3BITS 0x07 83 #define _4BITS 0x0F 84 #define _8BITS 0xFF 85 86 #define MAGICNUMBER_SIZE 4 87 #define LZ4IO_MAGICNUMBER 0x184D2204 88 #define LZ4IO_SKIPPABLE0 0x184D2A50 89 #define LZ4IO_SKIPPABLEMASK 0xFFFFFFF0 90 #define LEGACY_MAGICNUMBER 0x184C2102 91 92 #define CACHELINE 64 93 #define LEGACY_BLOCKSIZE (8 MB) 94 #define MIN_STREAM_BUFSIZE (192 KB) 95 #define LZ4IO_BLOCKSIZEID_DEFAULT 7 96 97 98 /************************************** 99 * Macros 100 **************************************/ 101 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 102 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } 103 static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ 104 105 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ 106 if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \ 107 { g_time = clock(); DISPLAY(__VA_ARGS__); \ 108 if (g_displayLevel>=4) fflush(stderr); } } 109 static const clock_t refreshRate = CLOCKS_PER_SEC / 6; 110 static clock_t g_time = 0; 111 112 113 /************************************** 114 * Local Parameters 115 **************************************/ 116 static int g_overwrite = 1; 117 static int g_testMode = 0; 118 static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; 119 static int g_blockChecksum = 0; 120 static int g_streamChecksum = 1; 121 static int g_blockIndependence = 1; 122 static int g_sparseFileSupport = 1; 123 static int g_contentSizeFlag = 0; 124 125 126 /************************************** 127 * Exceptions 128 ***************************************/ 129 #ifndef DEBUG 130 # define DEBUG 0 131 #endif 132 #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); 133 #define EXM_THROW(error, ...) \ 134 { \ 135 DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 136 DISPLAYLEVEL(1, "Error %i : ", error); \ 137 DISPLAYLEVEL(1, __VA_ARGS__); \ 138 DISPLAYLEVEL(1, " \n"); \ 139 exit(error); \ 140 } 141 142 143 /************************************** 144 * Version modifiers 145 **************************************/ 146 #define EXTENDED_ARGUMENTS 147 #define EXTENDED_HELP 148 #define EXTENDED_FORMAT 149 #define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F 150 151 152 /* ************************************************** */ 153 /* ****************** Parameters ******************** */ 154 /* ************************************************** */ 155 156 /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ 157 int LZ4IO_setOverwrite(int yes) 158 { 159 g_overwrite = (yes!=0); 160 return g_overwrite; 161 } 162 163 /* Default setting : testMode = 0; return : testMode (0/1) */ 164 int LZ4IO_setTestMode(int yes) 165 { 166 g_testMode = (yes!=0); 167 return g_testMode; 168 } 169 170 /* blockSizeID : valid values : 4-5-6-7 */ 171 size_t LZ4IO_setBlockSizeID(unsigned bsid) 172 { 173 static const size_t blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; 174 static const unsigned minBlockSizeID = 4; 175 static const unsigned maxBlockSizeID = 7; 176 if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; 177 g_blockSizeId = bsid; 178 return blockSizeTable[g_blockSizeId-minBlockSizeID]; 179 } 180 181 int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) 182 { 183 g_blockIndependence = (blockMode == LZ4IO_blockIndependent); 184 return g_blockIndependence; 185 } 186 187 /* Default setting : no checksum */ 188 int LZ4IO_setBlockChecksumMode(int xxhash) 189 { 190 g_blockChecksum = (xxhash != 0); 191 return g_blockChecksum; 192 } 193 194 /* Default setting : checksum enabled */ 195 int LZ4IO_setStreamChecksumMode(int xxhash) 196 { 197 g_streamChecksum = (xxhash != 0); 198 return g_streamChecksum; 199 } 200 201 /* Default setting : 0 (no notification) */ 202 int LZ4IO_setNotificationLevel(int level) 203 { 204 g_displayLevel = level; 205 return g_displayLevel; 206 } 207 208 /* Default setting : 0 (disabled) */ 209 int LZ4IO_setSparseFile(int enable) 210 { 211 g_sparseFileSupport = (enable!=0); 212 return g_sparseFileSupport; 213 } 214 215 /* Default setting : 0 (disabled) */ 216 int LZ4IO_setContentSize(int enable) 217 { 218 g_contentSizeFlag = (enable!=0); 219 return g_contentSizeFlag; 220 } 221 222 static U32 g_removeSrcFile = 0; 223 void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } 224 225 226 227 /* ************************************************************************ ** 228 ** ********************** LZ4 File / Pipe compression ********************* ** 229 ** ************************************************************************ */ 230 231 static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 232 static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } 233 234 235 /** LZ4IO_openSrcFile() : 236 * condition : `dstFileName` must be non-NULL. 237 * @result : FILE* to `dstFileName`, or NULL if it fails */ 238 static FILE* LZ4IO_openSrcFile(const char* srcFileName) 239 { 240 FILE* f; 241 242 if (!strcmp (srcFileName, stdinmark)) { 243 DISPLAYLEVEL(4,"Using stdin for input\n"); 244 f = stdin; 245 SET_BINARY_MODE(stdin); 246 } else { 247 f = fopen(srcFileName, "rb"); 248 if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno)); 249 } 250 251 return f; 252 } 253 254 /** FIO_openDstFile() : 255 * condition : `dstFileName` must be non-NULL. 256 * @result : FILE* to `dstFileName`, or NULL if it fails */ 257 static FILE* LZ4IO_openDstFile(const char* dstFileName) 258 { 259 FILE* f; 260 261 if (!strcmp (dstFileName, stdoutmark)) { 262 DISPLAYLEVEL(4,"Using stdout for output\n"); 263 f = stdout; 264 SET_BINARY_MODE(stdout); 265 if (g_sparseFileSupport==1) { 266 g_sparseFileSupport = 0; 267 DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); 268 } 269 } else { 270 if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ 271 f = fopen( dstFileName, "rb" ); 272 if (f != NULL) { /* dest exists, prompt for overwrite authorization */ 273 fclose(f); 274 if (g_displayLevel <= 1) { /* No interaction possible */ 275 DISPLAY("%s already exists; not overwritten \n", dstFileName); 276 return NULL; 277 } 278 DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName); 279 { int ch = getchar(); 280 if ((ch!='Y') && (ch!='y')) { 281 DISPLAY(" not overwritten \n"); 282 return NULL; 283 } 284 while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ 285 } } } 286 f = fopen( dstFileName, "wb" ); 287 if (f==NULL) DISPLAYLEVEL(1, "%s: %s\n", dstFileName, strerror(errno)); 288 } 289 290 /* sparse file */ 291 if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } 292 293 return f; 294 } 295 296 297 298 /*************************************** 299 * Legacy Compression 300 ***************************************/ 301 302 /* unoptimized version; solves endianess & alignment issues */ 303 static void LZ4IO_writeLE32 (void* p, unsigned value32) 304 { 305 unsigned char* dstPtr = (unsigned char*)p; 306 dstPtr[0] = (unsigned char)value32; 307 dstPtr[1] = (unsigned char)(value32 >> 8); 308 dstPtr[2] = (unsigned char)(value32 >> 16); 309 dstPtr[3] = (unsigned char)(value32 >> 24); 310 } 311 312 static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel) 313 { 314 (void)cLevel; 315 return LZ4_compress_fast(src, dst, srcSize, dstSize, 1); 316 } 317 318 /* LZ4IO_compressFilename_Legacy : 319 * This function is intentionally "hidden" (not published in .h) 320 * It generates compressed streams using the old 'legacy' format */ 321 int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) 322 { 323 int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); 324 unsigned long long filesize = 0; 325 unsigned long long compressedfilesize = MAGICNUMBER_SIZE; 326 char* in_buff; 327 char* out_buff; 328 const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); 329 FILE* finput; 330 FILE* foutput; 331 clock_t end; 332 333 /* Init */ 334 clock_t const start = clock(); 335 if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC; 336 337 finput = LZ4IO_openSrcFile(input_filename); 338 if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename); 339 foutput = LZ4IO_openDstFile(output_filename); 340 if (foutput == NULL) { fclose(finput); EXM_THROW(20, "%s : open file error ", input_filename); } 341 342 /* Allocate Memory */ 343 in_buff = (char*)malloc(LEGACY_BLOCKSIZE); 344 out_buff = (char*)malloc(outBuffSize); 345 if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); 346 347 /* Write Archive Header */ 348 LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); 349 { size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); 350 if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); } 351 352 /* Main Loop */ 353 while (1) { 354 unsigned int outSize; 355 /* Read Block */ 356 size_t const inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); 357 if (inSize == 0) break; 358 if (inSize > LEGACY_BLOCKSIZE) EXM_THROW(23, "Read error : wrong fread() size report "); /* should be impossible */ 359 filesize += inSize; 360 361 /* Compress Block */ 362 outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel); 363 compressedfilesize += outSize+4; 364 DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); 365 366 /* Write Block */ 367 LZ4IO_writeLE32(out_buff, outSize); 368 { size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); 369 if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(24, "Write error : cannot write compressed block"); 370 } } 371 if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename); 372 373 /* Status */ 374 end = clock(); 375 if (end==start) end+=1; /* avoid division by zero (speed) */ 376 filesize += !filesize; /* avoid division by zero (ratio) */ 377 DISPLAYLEVEL(2, "\r%79s\r", ""); /* blank line */ 378 DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", 379 filesize, compressedfilesize, (double)compressedfilesize / filesize * 100); 380 { double const seconds = (double)(end - start) / CLOCKS_PER_SEC; 381 DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); 382 } 383 384 /* Close & Free */ 385 free(in_buff); 386 free(out_buff); 387 fclose(finput); 388 fclose(foutput); 389 390 return 0; 391 } 392 393 394 /********************************************* 395 * Compression using Frame format 396 *********************************************/ 397 398 typedef struct { 399 void* srcBuffer; 400 size_t srcBufferSize; 401 void* dstBuffer; 402 size_t dstBufferSize; 403 LZ4F_compressionContext_t ctx; 404 } cRess_t; 405 406 static cRess_t LZ4IO_createCResources(void) 407 { 408 const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); 409 cRess_t ress; 410 411 LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); 412 if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); 413 414 /* Allocate Memory */ 415 ress.srcBuffer = malloc(blockSize); 416 ress.srcBufferSize = blockSize; 417 ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */ 418 ress.dstBuffer = malloc(ress.dstBufferSize); 419 if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); 420 421 return ress; 422 } 423 424 static void LZ4IO_freeCResources(cRess_t ress) 425 { 426 free(ress.srcBuffer); 427 free(ress.dstBuffer); 428 { LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx); 429 if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); } 430 } 431 432 /* 433 * LZ4IO_compressFilename_extRess() 434 * result : 0 : compression completed correctly 435 * 1 : missing or pb opening srcFileName 436 */ 437 static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) 438 { 439 unsigned long long filesize = 0; 440 unsigned long long compressedfilesize = 0; 441 FILE* srcFile; 442 FILE* dstFile; 443 void* const srcBuffer = ress.srcBuffer; 444 void* const dstBuffer = ress.dstBuffer; 445 const size_t dstBufferSize = ress.dstBufferSize; 446 const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); 447 size_t readSize; 448 LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ 449 LZ4F_preferences_t prefs; 450 451 /* Init */ 452 srcFile = LZ4IO_openSrcFile(srcFileName); 453 if (srcFile == NULL) return 1; 454 dstFile = LZ4IO_openDstFile(dstFileName); 455 if (dstFile == NULL) { fclose(srcFile); return 1; } 456 memset(&prefs, 0, sizeof(prefs)); 457 458 459 /* Set compression parameters */ 460 prefs.autoFlush = 1; 461 prefs.compressionLevel = compressionLevel; 462 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; 463 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; 464 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; 465 if (g_contentSizeFlag) { 466 U64 const fileSize = UTIL_getFileSize(srcFileName); 467 prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ 468 if (fileSize==0) 469 DISPLAYLEVEL(3, "Warning : cannot determine input content size \n"); 470 } 471 472 /* read first block */ 473 readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); 474 if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName); 475 filesize += readSize; 476 477 /* single-block file */ 478 if (readSize < blockSize) { 479 /* Compress in single pass */ 480 size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); 481 if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize)); 482 compressedfilesize = cSize; 483 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", 484 (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ 485 486 /* Write Block */ 487 { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); 488 if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); 489 } } 490 491 else 492 493 /* multiple-blocks file */ 494 { 495 /* Write Archive Header */ 496 size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); 497 if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); 498 { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); 499 if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } 500 compressedfilesize += headerSize; 501 502 /* Main Loop */ 503 while (readSize>0) { 504 size_t outSize; 505 506 /* Compress Block */ 507 outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); 508 if (LZ4F_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize)); 509 compressedfilesize += outSize; 510 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); 511 512 /* Write Block */ 513 { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); 514 if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } 515 516 /* Read next block */ 517 readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); 518 filesize += readSize; 519 } 520 if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); 521 522 /* End of Stream mark */ 523 headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); 524 if (LZ4F_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); 525 526 { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); 527 if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } 528 compressedfilesize += headerSize; 529 } 530 531 /* Release files */ 532 fclose (srcFile); 533 fclose (dstFile); 534 535 /* Copy owner, file permissions and modification time */ 536 { stat_t statbuf; 537 if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf)) 538 UTIL_setFileStat(dstFileName, &statbuf); 539 } 540 541 if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ 542 543 /* Final Status */ 544 DISPLAYLEVEL(2, "\r%79s\r", ""); 545 DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", 546 filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ 547 548 return 0; 549 } 550 551 552 int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) 553 { 554 clock_t const start = clock(); 555 cRess_t const ress = LZ4IO_createCResources(); 556 557 int const issueWithSrcFile = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); 558 559 /* Free resources */ 560 LZ4IO_freeCResources(ress); 561 562 /* Final Status */ 563 { clock_t const end = clock(); 564 double const seconds = (double)(end - start) / CLOCKS_PER_SEC; 565 DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); 566 } 567 568 return issueWithSrcFile; 569 } 570 571 572 #define FNSPACE 30 573 int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) 574 { 575 int i; 576 int missed_files = 0; 577 char* dstFileName = (char*)malloc(FNSPACE); 578 size_t ofnSize = FNSPACE; 579 const size_t suffixSize = strlen(suffix); 580 cRess_t ress; 581 582 if (dstFileName == NULL) return ifntSize; /* not enough memory */ 583 ress = LZ4IO_createCResources(); 584 585 /* loop on each file */ 586 for (i=0; i<ifntSize; i++) { 587 size_t const ifnSize = strlen(inFileNamesTable[i]); 588 if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); if (dstFileName==NULL) { LZ4IO_freeCResources(ress); return ifntSize; } } 589 strcpy(dstFileName, inFileNamesTable[i]); 590 strcat(dstFileName, suffix); 591 592 missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel); 593 } 594 595 /* Close & Free */ 596 LZ4IO_freeCResources(ress); 597 free(dstFileName); 598 599 return missed_files; 600 } 601 602 603 /* ********************************************************************* */ 604 /* ********************** LZ4 file-stream Decompression **************** */ 605 /* ********************************************************************* */ 606 607 static unsigned LZ4IO_readLE32 (const void* s) 608 { 609 const unsigned char* const srcPtr = (const unsigned char*)s; 610 unsigned value32 = srcPtr[0]; 611 value32 += (srcPtr[1]<<8); 612 value32 += (srcPtr[2]<<16); 613 value32 += ((unsigned)srcPtr[3])<<24; 614 return value32; 615 } 616 617 #define sizeT sizeof(size_t) 618 #define maskT (sizeT - 1) 619 620 static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) 621 { 622 const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ 623 const size_t* ptrT = bufferT; 624 size_t bufferSizeT = bufferSize / sizeT; 625 const size_t* const bufferTEnd = bufferT + bufferSizeT; 626 static const size_t segmentSizeT = (32 KB) / sizeT; 627 628 if (!g_sparseFileSupport) { /* normal write */ 629 size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); 630 if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); 631 return 0; 632 } 633 634 /* avoid int overflow */ 635 if (storedSkips > 1 GB) { 636 int const seekResult = fseek(file, 1 GB, SEEK_CUR); 637 if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); 638 storedSkips -= 1 GB; 639 } 640 641 while (ptrT < bufferTEnd) { 642 size_t seg0SizeT = segmentSizeT; 643 size_t nb0T; 644 645 /* count leading zeros */ 646 if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; 647 bufferSizeT -= seg0SizeT; 648 for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; 649 storedSkips += (unsigned)(nb0T * sizeT); 650 651 if (nb0T != seg0SizeT) { /* not all 0s */ 652 errno = 0; 653 { int const seekResult = fseek(file, storedSkips, SEEK_CUR); 654 if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); 655 } 656 storedSkips = 0; 657 seg0SizeT -= nb0T; 658 ptrT += nb0T; 659 { size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); 660 if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); 661 } } 662 ptrT += seg0SizeT; 663 } 664 665 if (bufferSize & maskT) { /* size not multiple of sizeT : implies end of block */ 666 const char* const restStart = (const char*)bufferTEnd; 667 const char* restPtr = restStart; 668 size_t const restSize = bufferSize & maskT; 669 const char* const restEnd = restStart + restSize; 670 for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; 671 storedSkips += (unsigned) (restPtr - restStart); 672 if (restPtr != restEnd) { 673 int const seekResult = fseek(file, storedSkips, SEEK_CUR); 674 if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); 675 storedSkips = 0; 676 { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); 677 if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); 678 } } 679 } 680 681 return storedSkips; 682 } 683 684 static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) 685 { 686 if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ 687 int const seekResult = fseek(file, storedSkips-1, SEEK_CUR); 688 if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); 689 { const char lastZeroByte[1] = { 0 }; 690 size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); 691 if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); 692 } } 693 } 694 695 696 static unsigned g_magicRead = 0; 697 static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) 698 { 699 unsigned long long filesize = 0; 700 char* in_buff; 701 char* out_buff; 702 unsigned storedSkips = 0; 703 704 /* Allocate Memory */ 705 in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); 706 out_buff = (char*)malloc(LEGACY_BLOCKSIZE); 707 if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); 708 709 /* Main Loop */ 710 while (1) { 711 int decodeSize; 712 unsigned int blockSize; 713 714 /* Block Size */ 715 { size_t const sizeCheck = fread(in_buff, 1, 4, finput); 716 if (sizeCheck == 0) break; /* Nothing to read : file read is completed */ 717 if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); } 718 blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ 719 if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) { 720 /* Cannot read next block : maybe new stream ? */ 721 g_magicRead = blockSize; 722 break; 723 } 724 725 /* Read Block */ 726 { size_t const sizeCheck = fread(in_buff, 1, blockSize, finput); 727 if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); } 728 729 /* Decode Block */ 730 decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); 731 if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); 732 filesize += decodeSize; 733 734 /* Write Block */ 735 storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); 736 } 737 if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); 738 739 LZ4IO_fwriteSparseEnd(foutput, storedSkips); 740 741 /* Free */ 742 free(in_buff); 743 free(out_buff); 744 745 return filesize; 746 } 747 748 749 750 typedef struct { 751 void* srcBuffer; 752 size_t srcBufferSize; 753 void* dstBuffer; 754 size_t dstBufferSize; 755 FILE* dstFile; 756 LZ4F_decompressionContext_t dCtx; 757 } dRess_t; 758 759 static const size_t LZ4IO_dBufferSize = 64 KB; 760 static dRess_t LZ4IO_createDResources(void) 761 { 762 dRess_t ress; 763 764 /* init */ 765 LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); 766 if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); 767 768 /* Allocate Memory */ 769 ress.srcBufferSize = LZ4IO_dBufferSize; 770 ress.srcBuffer = malloc(ress.srcBufferSize); 771 ress.dstBufferSize = LZ4IO_dBufferSize; 772 ress.dstBuffer = malloc(ress.dstBufferSize); 773 if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); 774 775 ress.dstFile = NULL; 776 return ress; 777 } 778 779 static void LZ4IO_freeDResources(dRess_t ress) 780 { 781 LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx); 782 if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); 783 free(ress.srcBuffer); 784 free(ress.dstBuffer); 785 } 786 787 788 static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) 789 { 790 unsigned long long filesize = 0; 791 LZ4F_errorCode_t nextToLoad; 792 unsigned storedSkips = 0; 793 794 /* Init feed with magic number (already consumed from FILE* sFile) */ 795 { size_t inSize = MAGICNUMBER_SIZE; 796 size_t outSize= 0; 797 LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); 798 nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); 799 if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); 800 } 801 802 /* Main Loop */ 803 for (;nextToLoad;) { 804 size_t readSize; 805 size_t pos = 0; 806 size_t decodedBytes = ress.dstBufferSize; 807 808 /* Read input */ 809 if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize; 810 readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile); 811 if (!readSize) break; /* reached end of file or stream */ 812 813 while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) { /* still to read, or still to flush */ 814 /* Decode Input (at least partially) */ 815 size_t remaining = readSize - pos; 816 decodedBytes = ress.dstBufferSize; 817 nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); 818 if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); 819 pos += remaining; 820 821 /* Write Block */ 822 if (decodedBytes) { 823 if (!g_testMode) 824 storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); 825 filesize += decodedBytes; 826 DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); 827 } 828 829 if (!nextToLoad) break; 830 } 831 } 832 /* can be out because readSize == 0, which could be an fread() error */ 833 if (ferror(srcFile)) EXM_THROW(67, "Read error"); 834 835 if (!g_testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); 836 if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); 837 838 return filesize; 839 } 840 841 842 #define PTSIZE (64 KB) 843 #define PTSIZET (PTSIZE / sizeof(size_t)) 844 static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) 845 { 846 size_t buffer[PTSIZET]; 847 size_t readBytes = 1; 848 unsigned long long total = MAGICNUMBER_SIZE; 849 unsigned storedSkips = 0; 850 851 size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); 852 if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); 853 854 while (readBytes) { 855 readBytes = fread(buffer, 1, PTSIZE, finput); 856 total += readBytes; 857 storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips); 858 } 859 if (ferror(finput)) EXM_THROW(51, "Read Error") 860 861 LZ4IO_fwriteSparseEnd(foutput, storedSkips); 862 return total; 863 } 864 865 866 /** Safely handle cases when (unsigned)offset > LONG_MAX */ 867 static int fseek_u32(FILE *fp, unsigned offset, int where) 868 { 869 const unsigned stepMax = 1U << 30; 870 int errorNb = 0; 871 872 if (where != SEEK_CUR) return -1; /* Only allows SEEK_CUR */ 873 while (offset > 0) { 874 unsigned s = offset; 875 if (s > stepMax) s = stepMax; 876 errorNb = fseek(fp, (long) s, SEEK_CUR); 877 if (errorNb != 0) break; 878 offset -= s; 879 } 880 return errorNb; 881 } 882 883 #define ENDOFSTREAM ((unsigned long long)-1) 884 static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) 885 { 886 unsigned char MNstore[MAGICNUMBER_SIZE]; 887 unsigned magicNumber; 888 static unsigned nbCalls = 0; 889 890 /* init */ 891 nbCalls++; 892 893 /* Check Archive Header */ 894 if (g_magicRead) { /* magic number already read from finput (see legacy frame)*/ 895 magicNumber = g_magicRead; 896 g_magicRead = 0; 897 } else { 898 size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); 899 if (nbReadBytes==0) { nbCalls = 0; return ENDOFSTREAM; } /* EOF */ 900 if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); 901 magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ 902 } 903 if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ 904 905 switch(magicNumber) 906 { 907 case LZ4IO_MAGICNUMBER: 908 return LZ4IO_decompressLZ4F(ress, finput, foutput); 909 case LEGACY_MAGICNUMBER: 910 DISPLAYLEVEL(4, "Detected : Legacy format \n"); 911 return LZ4IO_decodeLegacyStream(finput, foutput); 912 case LZ4IO_SKIPPABLE0: 913 DISPLAYLEVEL(4, "Skipping detected skippable area \n"); 914 { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); 915 if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); } 916 { unsigned const size = LZ4IO_readLE32(MNstore); /* Little Endian format */ 917 int const errorNb = fseek_u32(finput, size, SEEK_CUR); 918 if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); } 919 return 0; 920 EXTENDED_FORMAT; /* macro extension for custom formats */ 921 default: 922 if (nbCalls == 1) { /* just started */ 923 if (!g_testMode && g_overwrite) { 924 nbCalls = 0; 925 return LZ4IO_passThrough(finput, foutput, MNstore); 926 } 927 EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ 928 } 929 DISPLAYLEVEL(2, "Stream followed by undecodable data\n"); 930 return ENDOFSTREAM; 931 } 932 } 933 934 935 static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) 936 { 937 FILE* const foutput = ress.dstFile; 938 unsigned long long filesize = 0, decodedSize=0; 939 FILE* finput; 940 941 /* Init */ 942 finput = LZ4IO_openSrcFile(input_filename); 943 if (finput==NULL) return 1; 944 945 /* Loop over multiple streams */ 946 do { 947 decodedSize = selectDecoder(ress, finput, foutput); 948 if (decodedSize != ENDOFSTREAM) 949 filesize += decodedSize; 950 } while (decodedSize != ENDOFSTREAM); 951 952 /* Close */ 953 fclose(finput); 954 955 if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* remove source file : --rm */ 956 957 /* Final Status */ 958 DISPLAYLEVEL(2, "\r%79s\r", ""); 959 DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize); 960 (void)output_filename; 961 962 return 0; 963 } 964 965 966 static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) 967 { 968 FILE* foutput; 969 970 /* Init */ 971 foutput = LZ4IO_openDstFile(output_filename); 972 if (foutput==NULL) return 1; /* failure */ 973 974 ress.dstFile = foutput; 975 LZ4IO_decompressSrcFile(ress, input_filename, output_filename); 976 977 fclose(foutput); 978 979 /* Copy owner, file permissions and modification time */ 980 { stat_t statbuf; 981 if (strcmp (input_filename, stdinmark) && strcmp (output_filename, stdoutmark) && UTIL_getFileStat(input_filename, &statbuf)) 982 UTIL_setFileStat(output_filename, &statbuf); 983 } 984 985 return 0; 986 } 987 988 989 int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) 990 { 991 dRess_t const ress = LZ4IO_createDResources(); 992 clock_t const start = clock(); 993 994 int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename); 995 996 { clock_t const end = clock(); 997 double const seconds = (double)(end - start) / CLOCKS_PER_SEC; 998 DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); 999 } 1000 1001 LZ4IO_freeDResources(ress); 1002 return missingFiles; 1003 } 1004 1005 1006 int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) 1007 { 1008 int i; 1009 int skippedFiles = 0; 1010 int missingFiles = 0; 1011 char* outFileName = (char*)malloc(FNSPACE); 1012 size_t ofnSize = FNSPACE; 1013 size_t const suffixSize = strlen(suffix); 1014 dRess_t ress = LZ4IO_createDResources(); 1015 1016 if (outFileName==NULL) return ifntSize; /* not enough memory */ 1017 ress.dstFile = LZ4IO_openDstFile(stdoutmark); 1018 1019 for (i=0; i<ifntSize; i++) { 1020 size_t const ifnSize = strlen(inFileNamesTable[i]); 1021 const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; 1022 if (!strcmp(suffix, stdoutmark)) { 1023 missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark); 1024 continue; 1025 } 1026 if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; } 1027 if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { 1028 DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); 1029 skippedFiles++; 1030 continue; 1031 } 1032 memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); 1033 outFileName[ifnSize-suffixSize] = '\0'; 1034 missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName); 1035 } 1036 1037 LZ4IO_freeDResources(ress); 1038 free(outFileName); 1039 return missingFiles + skippedFiles; 1040 } 1041
