1 /* LZ4frame API example : compress a file 2 * Modified from an example code by Zbigniew Jdrzejewski-Szmek 3 * 4 * This example streams an input file into an output file 5 * using a bounded memory budget. 6 * Input is read in chunks of IN_CHUNK_SIZE */ 7 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <string.h> 11 #include <errno.h> 12 #include <assert.h> 13 14 #include <lz4frame.h> 15 16 17 #define IN_CHUNK_SIZE (16*1024) 18 19 static const LZ4F_preferences_t kPrefs = { 20 { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 21 0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum }, 22 0, /* compression level; 0 == default */ 23 0, /* autoflush */ 24 0, /* favor decompression speed */ 25 { 0, 0, 0 }, /* reserved, must be set to 0 */ 26 }; 27 28 29 /* safe_fwrite() : 30 * performs fwrite(), ensure operation success, or immediately exit() */ 31 static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f) 32 { 33 size_t const writtenSize = fwrite(buf, eltSize, nbElt, f); 34 size_t const expectedSize = eltSize * nbElt; 35 assert(expectedSize / nbElt == eltSize); /* check overflow */ 36 if (writtenSize < expectedSize) { 37 if (ferror(f)) /* note : ferror() must follow fwrite */ 38 fprintf(stderr, "Write failed \n"); 39 else 40 fprintf(stderr, "Short write \n"); 41 exit(1); 42 } 43 } 44 45 46 /* ================================================= */ 47 /* Streaming Compression example */ 48 /* ================================================= */ 49 50 typedef struct { 51 int error; 52 unsigned long long size_in; 53 unsigned long long size_out; 54 } compressResult_t; 55 56 static compressResult_t 57 compress_file_internal(FILE* f_in, FILE* f_out, 58 LZ4F_compressionContext_t ctx, 59 void* inBuff, size_t inChunkSize, 60 void* outBuff, size_t outCapacity) 61 { 62 compressResult_t result = { 1, 0, 0 }; /* result for an error */ 63 unsigned long long count_in = 0, count_out; 64 65 assert(f_in != NULL); assert(f_out != NULL); 66 assert(ctx != NULL); 67 assert(outCapacity >= LZ4F_HEADER_SIZE_MAX); 68 assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs)); 69 70 /* write frame header */ 71 { size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs); 72 if (LZ4F_isError(headerSize)) { 73 printf("Failed to start compression: error %zu\n", headerSize); 74 return result; 75 } 76 count_out = headerSize; 77 printf("Buffer size is %zu bytes, header size %zu bytes\n", outCapacity, headerSize); 78 safe_fwrite(outBuff, 1, headerSize, f_out); 79 } 80 81 /* stream file */ 82 for (;;) { 83 size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in); 84 if (readSize == 0) break; /* nothing left to read from input file */ 85 count_in += readSize; 86 87 size_t const compressedSize = LZ4F_compressUpdate(ctx, 88 outBuff, outCapacity, 89 inBuff, readSize, 90 NULL); 91 if (LZ4F_isError(compressedSize)) { 92 printf("Compression failed: error %zu\n", compressedSize); 93 return result; 94 } 95 96 printf("Writing %zu bytes\n", compressedSize); 97 safe_fwrite(outBuff, 1, compressedSize, f_out); 98 count_out += compressedSize; 99 } 100 101 /* flush whatever remains within internal buffers */ 102 { size_t const compressedSize = LZ4F_compressEnd(ctx, 103 outBuff, outCapacity, 104 NULL); 105 if (LZ4F_isError(compressedSize)) { 106 printf("Failed to end compression: error %zu\n", compressedSize); 107 return result; 108 } 109 110 printf("Writing %zu bytes\n", compressedSize); 111 safe_fwrite(outBuff, 1, compressedSize, f_out); 112 count_out += compressedSize; 113 } 114 115 result.size_in = count_in; 116 result.size_out = count_out; 117 result.error = 0; 118 return result; 119 } 120 121 static compressResult_t 122 compress_file(FILE* f_in, FILE* f_out) 123 { 124 assert(f_in != NULL); 125 assert(f_out != NULL); 126 127 /* ressource allocation */ 128 LZ4F_compressionContext_t ctx; 129 size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 130 void* const src = malloc(IN_CHUNK_SIZE); 131 size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs); /* large enough for any input <= IN_CHUNK_SIZE */ 132 void* const outbuff = malloc(outbufCapacity); 133 134 compressResult_t result = { 1, 0, 0 }; /* == error (default) */ 135 if (!LZ4F_isError(ctxCreation) && src && outbuff) { 136 result = compress_file_internal(f_in, f_out, 137 ctx, 138 src, IN_CHUNK_SIZE, 139 outbuff, outbufCapacity); 140 } else { 141 printf("error : ressource allocation failed \n"); 142 } 143 144 LZ4F_freeCompressionContext(ctx); /* supports free on NULL */ 145 free(src); 146 free(outbuff); 147 return result; 148 } 149 150 151 /* ================================================= */ 152 /* Streaming decompression example */ 153 /* ================================================= */ 154 155 static size_t get_block_size(const LZ4F_frameInfo_t* info) { 156 switch (info->blockSizeID) { 157 case LZ4F_default: 158 case LZ4F_max64KB: return 1 << 16; 159 case LZ4F_max256KB: return 1 << 18; 160 case LZ4F_max1MB: return 1 << 20; 161 case LZ4F_max4MB: return 1 << 22; 162 default: 163 printf("Impossible with expected frame specification (<=v1.6.1)\n"); 164 exit(1); 165 } 166 } 167 168 /* @return : 1==error, 0==success */ 169 static int 170 decompress_file_internal(FILE* f_in, FILE* f_out, 171 LZ4F_dctx* dctx, 172 void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed, 173 void* dst, size_t dstCapacity) 174 { 175 int firstChunk = 1; 176 size_t ret = 1; 177 178 assert(f_in != NULL); assert(f_out != NULL); 179 assert(dctx != NULL); 180 assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled); 181 assert(dst != NULL); assert(dstCapacity > 0); 182 183 /* Decompression */ 184 while (ret != 0) { 185 /* Load more input */ 186 size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0; 187 const void* srcPtr = src + alreadyConsumed; alreadyConsumed=0; 188 const void* const srcEnd = srcPtr + readSize; 189 if (readSize == 0 || ferror(f_in)) { 190 printf("Decompress: not enough input or error reading file\n"); 191 return 1; 192 } 193 194 /* Decompress: 195 * Continue while there is more input to read (srcPtr != srcEnd) 196 * and the frame isn't over (ret != 0) 197 */ 198 while (srcPtr < srcEnd && ret != 0) { 199 /* Any data within dst has been flushed at this stage */ 200 size_t dstSize = dstCapacity; 201 size_t srcSize = srcEnd - srcPtr; 202 ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); 203 if (LZ4F_isError(ret)) { 204 printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); 205 return 1; 206 } 207 /* Flush output */ 208 if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out); 209 /* Update input */ 210 srcPtr += srcSize; 211 } 212 213 assert(srcPtr <= srcEnd); 214 215 /* Ensure all input data has been consumed. 216 * It is valid to have multiple frames in the same file, 217 * but this example only supports one frame. 218 */ 219 if (srcPtr < srcEnd) { 220 printf("Decompress: Trailing data left in file after frame\n"); 221 return 1; 222 } 223 } 224 225 /* Check that there isn't trailing data in the file after the frame. 226 * It is valid to have multiple frames in the same file, 227 * but this example only supports one frame. 228 */ 229 { size_t const readSize = fread(src, 1, 1, f_in); 230 if (readSize != 0 || !feof(f_in)) { 231 printf("Decompress: Trailing data left in file after frame\n"); 232 return 1; 233 } } 234 235 return 0; 236 } 237 238 239 /* @return : 1==error, 0==completed */ 240 static int 241 decompress_file_allocDst(FILE* f_in, FILE* f_out, 242 LZ4F_dctx* dctx, 243 void* src, size_t srcCapacity) 244 { 245 assert(f_in != NULL); assert(f_out != NULL); 246 assert(dctx != NULL); 247 assert(src != NULL); 248 assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX); /* ensure LZ4F_getFrameInfo() can read enough data */ 249 250 /* Read Frame header */ 251 size_t const readSize = fread(src, 1, srcCapacity, f_in); 252 if (readSize == 0 || ferror(f_in)) { 253 printf("Decompress: not enough input or error reading file\n"); 254 return 1; 255 } 256 257 LZ4F_frameInfo_t info; 258 size_t consumedSize = readSize; 259 { size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize); 260 if (LZ4F_isError(fires)) { 261 printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires)); 262 return 1; 263 } } 264 265 /* Allocating enough space for an entire block isn't necessary for 266 * correctness, but it allows some memcpy's to be elided. 267 */ 268 size_t const dstCapacity = get_block_size(&info); 269 void* const dst = malloc(dstCapacity); 270 if (!dst) { perror("decompress_file(dst)"); return 1; } 271 272 int const decompressionResult = decompress_file_internal( 273 f_in, f_out, 274 dctx, 275 src, srcCapacity, readSize-consumedSize, consumedSize, 276 dst, dstCapacity); 277 278 free(dst); 279 return decompressionResult; 280 } 281 282 283 /* @result : 1==error, 0==success */ 284 static int decompress_file(FILE* f_in, FILE* f_out) 285 { 286 assert(f_in != NULL); assert(f_out != NULL); 287 288 /* Ressource allocation */ 289 void* const src = malloc(IN_CHUNK_SIZE); 290 if (!src) { perror("decompress_file(src)"); return 1; } 291 292 LZ4F_dctx* dctx; 293 { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); 294 if (LZ4F_isError(dctxStatus)) { 295 printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus)); 296 } } 297 298 int const result = !dctx ? 1 /* error */ : 299 decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE); 300 301 free(src); 302 LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ 303 return result; 304 } 305 306 307 int compareFiles(FILE* fp0, FILE* fp1) 308 { 309 int result = 0; 310 311 while (result==0) { 312 char b0[1024]; 313 char b1[1024]; 314 size_t const r0 = fread(b0, 1, sizeof(b0), fp0); 315 size_t const r1 = fread(b1, 1, sizeof(b1), fp1); 316 317 result = (r0 != r1); 318 if (!r0 || !r1) break; 319 if (!result) result = memcmp(b0, b1, r0); 320 } 321 322 return result; 323 } 324 325 326 int main(int argc, const char **argv) { 327 char inpFilename[256] = { 0 }; 328 char lz4Filename[256] = { 0 }; 329 char decFilename[256] = { 0 }; 330 331 if (argc < 2) { 332 printf("Please specify input filename\n"); 333 return 0; 334 } 335 336 snprintf(inpFilename, 256, "%s", argv[1]); 337 snprintf(lz4Filename, 256, "%s.lz4", argv[1]); 338 snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); 339 340 printf("inp = [%s]\n", inpFilename); 341 printf("lz4 = [%s]\n", lz4Filename); 342 printf("dec = [%s]\n", decFilename); 343 344 /* compress */ 345 { FILE* const inpFp = fopen(inpFilename, "rb"); 346 FILE* const outFp = fopen(lz4Filename, "wb"); 347 348 printf("compress : %s -> %s\n", inpFilename, lz4Filename); 349 compressResult_t const ret = compress_file(inpFp, outFp); 350 351 fclose(outFp); 352 fclose(inpFp); 353 354 if (ret.error) { 355 printf("compress : failed with code %i\n", ret.error); 356 return ret.error; 357 } 358 printf("%s: %zu %zu bytes, %.1f%%\n", 359 inpFilename, 360 (size_t)ret.size_in, (size_t)ret.size_out, /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */ 361 (double)ret.size_out / ret.size_in * 100); 362 printf("compress : done\n"); 363 } 364 365 /* decompress */ 366 { FILE* const inpFp = fopen(lz4Filename, "rb"); 367 FILE* const outFp = fopen(decFilename, "wb"); 368 369 printf("decompress : %s -> %s\n", lz4Filename, decFilename); 370 int const ret = decompress_file(inpFp, outFp); 371 372 fclose(outFp); 373 fclose(inpFp); 374 375 if (ret) { 376 printf("decompress : failed with code %i\n", ret); 377 return ret; 378 } 379 printf("decompress : done\n"); 380 } 381 382 /* verify */ 383 { FILE* const inpFp = fopen(inpFilename, "rb"); 384 FILE* const decFp = fopen(decFilename, "rb"); 385 386 printf("verify : %s <-> %s\n", inpFilename, decFilename); 387 int const cmp = compareFiles(inpFp, decFp); 388 389 fclose(decFp); 390 fclose(inpFp); 391 392 if (cmp) { 393 printf("corruption detected : decompressed file differs from original\n"); 394 return cmp; 395 } 396 printf("verify : OK\n"); 397 } 398 399 return 0; 400 } 401
