Home | History | Annotate | Download | only in examples 
      1 // LZ4 API example : Dictionary Random Access
      2 
      3 #if defined(_MSC_VER) && (_MSC_VER <= 1800)  /* Visual Studio <= 2013 */
      4 #  define _CRT_SECURE_NO_WARNINGS
      5 #  define snprintf sprintf_s
      6 #endif
      7 #include "lz4.h"
      8 
      9 #include <stdio.h>
     10 #include <stdint.h>
     11 #include <stdlib.h>
     12 #include <string.h>
     13 
     14 #define MIN(x, y) (x) < (y) ? (x) : (y)
     15 
     16 enum {
     17     BLOCK_BYTES = 1024,  /* 1 KiB of uncompressed data in a block */
     18     DICTIONARY_BYTES = 1024, /* Load a 1 KiB dictionary */
     19     MAX_BLOCKS = 1024 /* For simplicity of implementation */
     20 };
     21 
     22 /**
     23  * Magic bytes for this test case.
     24  * This is not a great magic number because it is a common word in ASCII.
     25  * However, it is important to have some versioning system in your format.
     26  */
     27 const char kTestMagic[] = { 'T', 'E', 'S', 'T' };
     28 
     29 
     30 void write_int(FILE* fp, int i) {
     31     size_t written = fwrite(&i, sizeof(i), 1, fp);
     32     if (written != 1) { exit(10); }
     33 }
     34 
     35 void write_bin(FILE* fp, const void* array, size_t arrayBytes) {
     36     size_t written = fwrite(array, 1, arrayBytes, fp);
     37     if (written != arrayBytes) { exit(11); }
     38 }
     39 
     40 void read_int(FILE* fp, int* i) {
     41     size_t read = fread(i, sizeof(*i), 1, fp);
     42     if (read != 1) { exit(12); }
     43 }
     44 
     45 size_t read_bin(FILE* fp, void* array, size_t arrayBytes) {
     46     size_t read = fread(array, 1, arrayBytes, fp);
     47     if (ferror(fp)) { exit(12); }
     48     return read;
     49 }
     50 
     51 void seek_bin(FILE* fp, long offset, int origin) {
     52     if (fseek(fp, offset, origin)) { exit(14); }
     53 }
     54 
     55 
     56 void test_compress(FILE* outFp, FILE* inpFp, void *dict, int dictSize)
     57 {
     58     LZ4_stream_t lz4Stream_body;
     59     LZ4_stream_t* lz4Stream = &lz4Stream_body;
     60 
     61     char inpBuf[BLOCK_BYTES];
     62     int offsets[MAX_BLOCKS];
     63     int *offsetsEnd = offsets;
     64 
     65 
     66     LZ4_resetStream(lz4Stream);
     67 
     68     /* Write header magic */
     69     write_bin(outFp, kTestMagic, sizeof(kTestMagic));
     70 
     71     *offsetsEnd++ = sizeof(kTestMagic);
     72     /* Write compressed data blocks.  Each block contains BLOCK_BYTES of plain
     73        data except possibly the last. */
     74     for(;;) {
     75         const int inpBytes = (int) read_bin(inpFp, inpBuf, BLOCK_BYTES);
     76         if(0 == inpBytes) {
     77             break;
     78         }
     79 
     80         /* Forget previously compressed data and load the dictionary */
     81         LZ4_loadDict(lz4Stream, dict, dictSize);
     82         {
     83             char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
     84             const int cmpBytes = LZ4_compress_fast_continue(
     85                 lz4Stream, inpBuf, cmpBuf, inpBytes, sizeof(cmpBuf), 1);
     86             if(cmpBytes <= 0) { exit(1); }
     87             write_bin(outFp, cmpBuf, (size_t)cmpBytes);
     88             /* Keep track of the offsets */
     89             *offsetsEnd = *(offsetsEnd - 1) + cmpBytes;
     90             ++offsetsEnd;
     91         }
     92         if (offsetsEnd - offsets > MAX_BLOCKS) { exit(2); }
     93     }
     94     /* Write the tailing jump table */
     95     {
     96         int *ptr = offsets;
     97         while (ptr != offsetsEnd) {
     98             write_int(outFp, *ptr++);
     99         }
    100         write_int(outFp, offsetsEnd - offsets);
    101     }
    102 }
    103 
    104 
    105 void test_decompress(FILE* outFp, FILE* inpFp, void *dict, int dictSize, int offset, int length)
    106 {
    107     LZ4_streamDecode_t lz4StreamDecode_body;
    108     LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
    109 
    110     /* The blocks [currentBlock, endBlock) contain the data we want */
    111     int currentBlock = offset / BLOCK_BYTES;
    112     int endBlock = ((offset + length - 1) / BLOCK_BYTES) + 1;
    113 
    114     char decBuf[BLOCK_BYTES];
    115     int offsets[MAX_BLOCKS];
    116 
    117     /* Special cases */
    118     if (length == 0) { return; }
    119 
    120     /* Read the magic bytes */
    121     {
    122         char magic[sizeof(kTestMagic)];
    123         size_t read = read_bin(inpFp, magic, sizeof(magic));
    124         if (read != sizeof(magic)) { exit(1); }
    125         if (memcmp(kTestMagic, magic, sizeof(magic))) { exit(2); }
    126     }
    127 
    128     /* Read the offsets tail */
    129     {
    130         int numOffsets;
    131         int block;
    132         int *offsetsPtr = offsets;
    133         seek_bin(inpFp, -4, SEEK_END);
    134         read_int(inpFp, &numOffsets);
    135         if (numOffsets <= endBlock) { exit(3); }
    136         seek_bin(inpFp, -4 * (numOffsets + 1), SEEK_END);
    137         for (block = 0; block <= endBlock; ++block) {
    138             read_int(inpFp, offsetsPtr++);
    139         }
    140     }
    141     /* Seek to the first block to read */
    142     seek_bin(inpFp, offsets[currentBlock], SEEK_SET);
    143     offset = offset % BLOCK_BYTES;
    144 
    145     /* Start decoding */
    146     for(; currentBlock < endBlock; ++currentBlock) {
    147         char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
    148         /* The difference in offsets is the size of the block */
    149         int  cmpBytes = offsets[currentBlock + 1] - offsets[currentBlock];
    150         {
    151             const size_t read = read_bin(inpFp, cmpBuf, (size_t)cmpBytes);
    152             if(read != (size_t)cmpBytes) { exit(4); }
    153         }
    154 
    155         /* Load the dictionary */
    156         LZ4_setStreamDecode(lz4StreamDecode, dict, dictSize);
    157         {
    158             const int decBytes = LZ4_decompress_safe_continue(
    159                 lz4StreamDecode, cmpBuf, decBuf, cmpBytes, BLOCK_BYTES);
    160             if(decBytes <= 0) { exit(5); }
    161             {
    162                 /* Write out the part of the data we care about */
    163                 int blockLength = MIN(length, (decBytes - offset));
    164                 write_bin(outFp, decBuf + offset, (size_t)blockLength);
    165                 offset = 0;
    166                 length -= blockLength;
    167             }
    168         }
    169     }
    170 }
    171 
    172 
    173 int compare(FILE* fp0, FILE* fp1, int length)
    174 {
    175     int result = 0;
    176 
    177     while(0 == result) {
    178         char b0[4096];
    179         char b1[4096];
    180         const size_t r0 = read_bin(fp0, b0, MIN(length, (int)sizeof(b0)));
    181         const size_t r1 = read_bin(fp1, b1, MIN(length, (int)sizeof(b1)));
    182 
    183         result = (int) r0 - (int) r1;
    184 
    185         if(0 == r0 || 0 == r1) {
    186             break;
    187         }
    188         if(0 == result) {
    189             result = memcmp(b0, b1, r0);
    190         }
    191         length -= r0;
    192     }
    193 
    194     return result;
    195 }
    196 
    197 
    198 int main(int argc, char* argv[])
    199 {
    200     char inpFilename[256] = { 0 };
    201     char lz4Filename[256] = { 0 };
    202     char decFilename[256] = { 0 };
    203     char dictFilename[256] = { 0 };
    204     int offset;
    205     int length;
    206     char dict[DICTIONARY_BYTES];
    207     int dictSize;
    208 
    209     if(argc < 5) {
    210         printf("Usage: %s input dictionary offset length", argv[0]);
    211         return 0;
    212     }
    213 
    214     snprintf(inpFilename, 256, "%s", argv[1]);
    215     snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], BLOCK_BYTES);
    216     snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], BLOCK_BYTES);
    217     snprintf(dictFilename, 256, "%s", argv[2]);
    218     offset = atoi(argv[3]);
    219     length = atoi(argv[4]);
    220 
    221     printf("inp    = [%s]\n", inpFilename);
    222     printf("lz4    = [%s]\n", lz4Filename);
    223     printf("dec    = [%s]\n", decFilename);
    224     printf("dict   = [%s]\n", dictFilename);
    225     printf("offset = [%d]\n", offset);
    226     printf("length = [%d]\n", length);
    227 
    228     /* Load dictionary */
    229     {
    230         FILE* dictFp = fopen(dictFilename, "rb");
    231         dictSize = (int)read_bin(dictFp, dict, DICTIONARY_BYTES);
    232         fclose(dictFp);
    233     }
    234 
    235     /* compress */
    236     {
    237         FILE* inpFp = fopen(inpFilename, "rb");
    238         FILE* outFp = fopen(lz4Filename, "wb");
    239 
    240         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
    241         test_compress(outFp, inpFp, dict, dictSize);
    242         printf("compress : done\n");
    243 
    244         fclose(outFp);
    245         fclose(inpFp);
    246     }
    247 
    248     /* decompress */
    249     {
    250         FILE* inpFp = fopen(lz4Filename, "rb");
    251         FILE* outFp = fopen(decFilename, "wb");
    252 
    253         printf("decompress : %s -> %s\n", lz4Filename, decFilename);
    254         test_decompress(outFp, inpFp, dict, DICTIONARY_BYTES, offset, length);
    255         printf("decompress : done\n");
    256 
    257         fclose(outFp);
    258         fclose(inpFp);
    259     }
    260 
    261     /* verify */
    262     {
    263         FILE* inpFp = fopen(inpFilename, "rb");
    264         FILE* decFp = fopen(decFilename, "rb");
    265         seek_bin(inpFp, offset, SEEK_SET);
    266 
    267         printf("verify : %s <-> %s\n", inpFilename, decFilename);
    268         const int cmp = compare(inpFp, decFp, length);
    269         if(0 == cmp) {
    270             printf("verify : OK\n");
    271         } else {
    272             printf("verify : NG\n");
    273         }
    274 
    275         fclose(decFp);
    276         fclose(inpFp);
    277     }
    278 
    279     return 0;
    280 }
    281