1 // LZ4 streaming API example : line-by-line logfile compression 2 // Copyright : Takayuki Matsuoka 3 4 5 #define _CRT_SECURE_NO_WARNINGS // for MSVC 6 #include "lz4.h" 7 8 #include <stdio.h> 9 #include <stdint.h> 10 #include <stdlib.h> 11 #include <string.h> 12 13 static size_t write_uint16(FILE* fp, uint16_t i) 14 { 15 return fwrite(&i, sizeof(i), 1, fp); 16 } 17 18 static size_t write_bin(FILE* fp, const void* array, int arrayBytes) 19 { 20 return fwrite(array, 1, arrayBytes, fp); 21 } 22 23 static size_t read_uint16(FILE* fp, uint16_t* i) 24 { 25 return fread(i, sizeof(*i), 1, fp); 26 } 27 28 static size_t read_bin(FILE* fp, void* array, int arrayBytes) 29 { 30 return fread(array, 1, arrayBytes, fp); 31 } 32 33 34 static void test_compress( 35 FILE* outFp, 36 FILE* inpFp, 37 size_t messageMaxBytes, 38 size_t ringBufferBytes) 39 { 40 LZ4_stream_t* const lz4Stream = LZ4_createStream(); 41 char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); 42 char* const inpBuf = malloc(ringBufferBytes); 43 int inpOffset = 0; 44 45 for ( ; ; ) 46 { 47 char* const inpPtr = &inpBuf[inpOffset]; 48 49 #if 0 50 // Read random length data to the ring buffer. 51 const int randomLength = (rand() % messageMaxBytes) + 1; 52 const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); 53 if (0 == inpBytes) break; 54 #else 55 // Read line to the ring buffer. 56 int inpBytes = 0; 57 if (!fgets(inpPtr, (int) messageMaxBytes, inpFp)) 58 break; 59 inpBytes = (int) strlen(inpPtr); 60 #endif 61 62 { 63 const int cmpBytes = LZ4_compress_continue( 64 lz4Stream, inpPtr, cmpBuf, inpBytes); 65 if (cmpBytes <= 0) break; 66 write_uint16(outFp, (uint16_t) cmpBytes); 67 write_bin(outFp, cmpBuf, cmpBytes); 68 69 // Add and wraparound the ringbuffer offset 70 inpOffset += inpBytes; 71 if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0; 72 } 73 } 74 write_uint16(outFp, 0); 75 76 free(inpBuf); 77 free(cmpBuf); 78 LZ4_freeStream(lz4Stream); 79 } 80 81 82 static void test_decompress( 83 FILE* outFp, 84 FILE* inpFp, 85 size_t messageMaxBytes, 86 size_t ringBufferBytes) 87 { 88 LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode(); 89 char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); 90 char* const decBuf = malloc(ringBufferBytes); 91 int decOffset = 0; 92 93 for ( ; ; ) 94 { 95 uint16_t cmpBytes = 0; 96 97 if (read_uint16(inpFp, &cmpBytes) != 1) break; 98 if (cmpBytes <= 0) break; 99 if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; 100 101 { 102 char* const decPtr = &decBuf[decOffset]; 103 const int decBytes = LZ4_decompress_safe_continue( 104 lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes); 105 if (decBytes <= 0) break; 106 write_bin(outFp, decPtr, decBytes); 107 108 // Add and wraparound the ringbuffer offset 109 decOffset += decBytes; 110 if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0; 111 } 112 } 113 114 free(decBuf); 115 free(cmpBuf); 116 LZ4_freeStreamDecode(lz4StreamDecode); 117 } 118 119 120 static int compare(FILE* f0, FILE* f1) 121 { 122 int result = 0; 123 const size_t tempBufferBytes = 65536; 124 char* const b0 = malloc(tempBufferBytes); 125 char* const b1 = malloc(tempBufferBytes); 126 127 while(0 == result) 128 { 129 const size_t r0 = fread(b0, 1, tempBufferBytes, f0); 130 const size_t r1 = fread(b1, 1, tempBufferBytes, f1); 131 132 result = (int) r0 - (int) r1; 133 134 if (0 == r0 || 0 == r1) break; 135 if (0 == result) result = memcmp(b0, b1, r0); 136 } 137 138 free(b1); 139 free(b0); 140 return result; 141 } 142 143 144 int main(int argc, char* argv[]) 145 { 146 enum { 147 MESSAGE_MAX_BYTES = 1024, 148 RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, 149 }; 150 151 char inpFilename[256] = { 0 }; 152 char lz4Filename[256] = { 0 }; 153 char decFilename[256] = { 0 }; 154 155 if (argc < 2) 156 { 157 printf("Please specify input filename\n"); 158 return 0; 159 } 160 161 snprintf(inpFilename, 256, "%s", argv[1]); 162 snprintf(lz4Filename, 256, "%s.lz4s", argv[1]); 163 snprintf(decFilename, 256, "%s.lz4s.dec", argv[1]); 164 165 printf("inp = [%s]\n", inpFilename); 166 printf("lz4 = [%s]\n", lz4Filename); 167 printf("dec = [%s]\n", decFilename); 168 169 // compress 170 { 171 FILE* inpFp = fopen(inpFilename, "rb"); 172 FILE* outFp = fopen(lz4Filename, "wb"); 173 174 test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); 175 176 fclose(outFp); 177 fclose(inpFp); 178 } 179 180 // decompress 181 { 182 FILE* inpFp = fopen(lz4Filename, "rb"); 183 FILE* outFp = fopen(decFilename, "wb"); 184 185 test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); 186 187 fclose(outFp); 188 fclose(inpFp); 189 } 190 191 // verify 192 { 193 FILE* inpFp = fopen(inpFilename, "rb"); 194 FILE* decFp = fopen(decFilename, "rb"); 195 196 const int cmp = compare(inpFp, decFp); 197 if (0 == cmp) 198 printf("Verify : OK\n"); 199 else 200 printf("Verify : NG\n"); 201 202 fclose(decFp); 203 fclose(inpFp); 204 } 205 206 return 0; 207 } 208
