1 // LZ4 streaming API example : line-by-line logfile compression 2 // Copyright : Takayuki Matsuoka 3 4 5 #ifdef _MSC_VER /* Visual Studio */ 6 # define _CRT_SECURE_NO_WARNINGS 7 # define snprintf sprintf_s 8 #endif 9 #include "lz4.h" 10 11 #include <stdio.h> 12 #include <stdint.h> 13 #include <stdlib.h> 14 #include <string.h> 15 16 static size_t write_uint16(FILE* fp, uint16_t i) 17 { 18 return fwrite(&i, sizeof(i), 1, fp); 19 } 20 21 static size_t write_bin(FILE* fp, const void* array, int arrayBytes) 22 { 23 return fwrite(array, 1, arrayBytes, fp); 24 } 25 26 static size_t read_uint16(FILE* fp, uint16_t* i) 27 { 28 return fread(i, sizeof(*i), 1, fp); 29 } 30 31 static size_t read_bin(FILE* fp, void* array, int arrayBytes) 32 { 33 return fread(array, 1, arrayBytes, fp); 34 } 35 36 37 static void test_compress( 38 FILE* outFp, 39 FILE* inpFp, 40 size_t messageMaxBytes, 41 size_t ringBufferBytes) 42 { 43 LZ4_stream_t* const lz4Stream = LZ4_createStream(); 44 const size_t cmpBufBytes = LZ4_COMPRESSBOUND(messageMaxBytes); 45 char* const cmpBuf = (char*) malloc(cmpBufBytes); 46 char* const inpBuf = (char*) malloc(ringBufferBytes); 47 int inpOffset = 0; 48 49 for ( ; ; ) 50 { 51 char* const inpPtr = &inpBuf[inpOffset]; 52 53 #if 0 54 // Read random length data to the ring buffer. 55 const int randomLength = (rand() % messageMaxBytes) + 1; 56 const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); 57 if (0 == inpBytes) break; 58 #else 59 // Read line to the ring buffer. 60 int inpBytes = 0; 61 if (!fgets(inpPtr, (int) messageMaxBytes, inpFp)) 62 break; 63 inpBytes = (int) strlen(inpPtr); 64 #endif 65 66 { 67 const int cmpBytes = LZ4_compress_fast_continue( 68 lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1); 69 if (cmpBytes <= 0) break; 70 write_uint16(outFp, (uint16_t) cmpBytes); 71 write_bin(outFp, cmpBuf, cmpBytes); 72 73 // Add and wraparound the ringbuffer offset 74 inpOffset += inpBytes; 75 if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0; 76 } 77 } 78 write_uint16(outFp, 0); 79 80 free(inpBuf); 81 free(cmpBuf); 82 LZ4_freeStream(lz4Stream); 83 } 84 85 86 static void test_decompress( 87 FILE* outFp, 88 FILE* inpFp, 89 size_t messageMaxBytes, 90 size_t ringBufferBytes) 91 { 92 LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode(); 93 char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); 94 char* const decBuf = (char*) malloc(ringBufferBytes); 95 int decOffset = 0; 96 97 for ( ; ; ) 98 { 99 uint16_t cmpBytes = 0; 100 101 if (read_uint16(inpFp, &cmpBytes) != 1) break; 102 if (cmpBytes <= 0) break; 103 if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; 104 105 { 106 char* const decPtr = &decBuf[decOffset]; 107 const int decBytes = LZ4_decompress_safe_continue( 108 lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes); 109 if (decBytes <= 0) break; 110 write_bin(outFp, decPtr, decBytes); 111 112 // Add and wraparound the ringbuffer offset 113 decOffset += decBytes; 114 if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0; 115 } 116 } 117 118 free(decBuf); 119 free(cmpBuf); 120 LZ4_freeStreamDecode(lz4StreamDecode); 121 } 122 123 124 static int compare(FILE* f0, FILE* f1) 125 { 126 int result = 0; 127 const size_t tempBufferBytes = 65536; 128 char* const b0 = (char*) malloc(tempBufferBytes); 129 char* const b1 = (char*) malloc(tempBufferBytes); 130 131 while(0 == result) 132 { 133 const size_t r0 = fread(b0, 1, tempBufferBytes, f0); 134 const size_t r1 = fread(b1, 1, tempBufferBytes, f1); 135 136 result = (int) r0 - (int) r1; 137 138 if (0 == r0 || 0 == r1) break; 139 if (0 == result) result = memcmp(b0, b1, r0); 140 } 141 142 free(b1); 143 free(b0); 144 return result; 145 } 146 147 148 int main(int argc, char* argv[]) 149 { 150 enum { 151 MESSAGE_MAX_BYTES = 1024, 152 RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, 153 }; 154 155 char inpFilename[256] = { 0 }; 156 char lz4Filename[256] = { 0 }; 157 char decFilename[256] = { 0 }; 158 159 if (argc < 2) 160 { 161 printf("Please specify input filename\n"); 162 return 0; 163 } 164 165 snprintf(inpFilename, 256, "%s", argv[1]); 166 snprintf(lz4Filename, 256, "%s.lz4s", argv[1]); 167 snprintf(decFilename, 256, "%s.lz4s.dec", argv[1]); 168 169 printf("inp = [%s]\n", inpFilename); 170 printf("lz4 = [%s]\n", lz4Filename); 171 printf("dec = [%s]\n", decFilename); 172 173 // compress 174 { 175 FILE* inpFp = fopen(inpFilename, "rb"); 176 FILE* outFp = fopen(lz4Filename, "wb"); 177 178 test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); 179 180 fclose(outFp); 181 fclose(inpFp); 182 } 183 184 // decompress 185 { 186 FILE* inpFp = fopen(lz4Filename, "rb"); 187 FILE* outFp = fopen(decFilename, "wb"); 188 189 test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); 190 191 fclose(outFp); 192 fclose(inpFp); 193 } 194 195 // verify 196 { 197 FILE* inpFp = fopen(inpFilename, "rb"); 198 FILE* decFp = fopen(decFilename, "rb"); 199 200 const int cmp = compare(inpFp, decFp); 201 if (0 == cmp) 202 printf("Verify : OK\n"); 203 else 204 printf("Verify : NG\n"); 205 206 fclose(decFp); 207 fclose(inpFp); 208 } 209 210 return 0; 211 } 212
