1 /* Lzma2Enc.c -- LZMA2 Encoder 2 2010-09-24 : Igor Pavlov : Public domain */ 3 4 /* #include <stdio.h> */ 5 #include <string.h> 6 7 /* #define _7ZIP_ST */ 8 9 #include "Lzma2Enc.h" 10 11 #ifndef _7ZIP_ST 12 #include "MtCoder.h" 13 #else 14 #define NUM_MT_CODER_THREADS_MAX 1 15 #endif 16 17 #define LZMA2_CONTROL_LZMA (1 << 7) 18 #define LZMA2_CONTROL_COPY_NO_RESET 2 19 #define LZMA2_CONTROL_COPY_RESET_DIC 1 20 #define LZMA2_CONTROL_EOF 0 21 22 #define LZMA2_LCLP_MAX 4 23 24 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) 25 26 #define LZMA2_PACK_SIZE_MAX (1 << 16) 27 #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX 28 #define LZMA2_UNPACK_SIZE_MAX (1 << 21) 29 #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX 30 31 #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16) 32 33 34 #define PRF(x) /* x */ 35 36 /* ---------- CLzma2EncInt ---------- */ 37 38 typedef struct 39 { 40 CLzmaEncHandle enc; 41 UInt64 srcPos; 42 Byte props; 43 Bool needInitState; 44 Bool needInitProp; 45 } CLzma2EncInt; 46 47 static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props) 48 { 49 Byte propsEncoded[LZMA_PROPS_SIZE]; 50 SizeT propsSize = LZMA_PROPS_SIZE; 51 RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); 52 RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); 53 p->srcPos = 0; 54 p->props = propsEncoded[0]; 55 p->needInitState = True; 56 p->needInitProp = True; 57 return SZ_OK; 58 } 59 60 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, 61 ISzAlloc *alloc, ISzAlloc *allocBig); 62 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, 63 UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig); 64 SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, 65 Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); 66 const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); 67 void LzmaEnc_Finish(CLzmaEncHandle pp); 68 void LzmaEnc_SaveState(CLzmaEncHandle pp); 69 void LzmaEnc_RestoreState(CLzmaEncHandle pp); 70 71 72 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, 73 size_t *packSizeRes, ISeqOutStream *outStream) 74 { 75 size_t packSizeLimit = *packSizeRes; 76 size_t packSize = packSizeLimit; 77 UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; 78 unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); 79 Bool useCopyBlock; 80 SRes res; 81 82 *packSizeRes = 0; 83 if (packSize < lzHeaderSize) 84 return SZ_ERROR_OUTPUT_EOF; 85 packSize -= lzHeaderSize; 86 87 LzmaEnc_SaveState(p->enc); 88 res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState, 89 outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize); 90 91 PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize)); 92 93 if (unpackSize == 0) 94 return res; 95 96 if (res == SZ_OK) 97 useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16)); 98 else 99 { 100 if (res != SZ_ERROR_OUTPUT_EOF) 101 return res; 102 res = SZ_OK; 103 useCopyBlock = True; 104 } 105 106 if (useCopyBlock) 107 { 108 size_t destPos = 0; 109 PRF(printf("################# COPY ")); 110 while (unpackSize > 0) 111 { 112 UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE; 113 if (packSizeLimit - destPos < u + 3) 114 return SZ_ERROR_OUTPUT_EOF; 115 outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET); 116 outBuf[destPos++] = (Byte)((u - 1) >> 8); 117 outBuf[destPos++] = (Byte)(u - 1); 118 memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u); 119 unpackSize -= u; 120 destPos += u; 121 p->srcPos += u; 122 if (outStream) 123 { 124 *packSizeRes += destPos; 125 if (outStream->Write(outStream, outBuf, destPos) != destPos) 126 return SZ_ERROR_WRITE; 127 destPos = 0; 128 } 129 else 130 *packSizeRes = destPos; 131 /* needInitState = True; */ 132 } 133 LzmaEnc_RestoreState(p->enc); 134 return SZ_OK; 135 } 136 { 137 size_t destPos = 0; 138 UInt32 u = unpackSize - 1; 139 UInt32 pm = (UInt32)(packSize - 1); 140 unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0); 141 142 PRF(printf(" ")); 143 144 outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F)); 145 outBuf[destPos++] = (Byte)(u >> 8); 146 outBuf[destPos++] = (Byte)u; 147 outBuf[destPos++] = (Byte)(pm >> 8); 148 outBuf[destPos++] = (Byte)pm; 149 150 if (p->needInitProp) 151 outBuf[destPos++] = p->props; 152 153 p->needInitProp = False; 154 p->needInitState = False; 155 destPos += packSize; 156 p->srcPos += unpackSize; 157 158 if (outStream) 159 if (outStream->Write(outStream, outBuf, destPos) != destPos) 160 return SZ_ERROR_WRITE; 161 *packSizeRes = destPos; 162 return SZ_OK; 163 } 164 } 165 166 /* ---------- Lzma2 Props ---------- */ 167 168 void Lzma2EncProps_Init(CLzma2EncProps *p) 169 { 170 LzmaEncProps_Init(&p->lzmaProps); 171 p->numTotalThreads = -1; 172 p->numBlockThreads = -1; 173 p->blockSize = 0; 174 } 175 176 void Lzma2EncProps_Normalize(CLzma2EncProps *p) 177 { 178 int t1, t1n, t2, t3; 179 { 180 CLzmaEncProps lzmaProps = p->lzmaProps; 181 LzmaEncProps_Normalize(&lzmaProps); 182 t1n = lzmaProps.numThreads; 183 } 184 185 t1 = p->lzmaProps.numThreads; 186 t2 = p->numBlockThreads; 187 t3 = p->numTotalThreads; 188 189 if (t2 > NUM_MT_CODER_THREADS_MAX) 190 t2 = NUM_MT_CODER_THREADS_MAX; 191 192 if (t3 <= 0) 193 { 194 if (t2 <= 0) 195 t2 = 1; 196 t3 = t1n * t2; 197 } 198 else if (t2 <= 0) 199 { 200 t2 = t3 / t1n; 201 if (t2 == 0) 202 { 203 t1 = 1; 204 t2 = t3; 205 } 206 if (t2 > NUM_MT_CODER_THREADS_MAX) 207 t2 = NUM_MT_CODER_THREADS_MAX; 208 } 209 else if (t1 <= 0) 210 { 211 t1 = t3 / t2; 212 if (t1 == 0) 213 t1 = 1; 214 } 215 else 216 t3 = t1n * t2; 217 218 p->lzmaProps.numThreads = t1; 219 p->numBlockThreads = t2; 220 p->numTotalThreads = t3; 221 LzmaEncProps_Normalize(&p->lzmaProps); 222 223 if (p->blockSize == 0) 224 { 225 UInt32 dictSize = p->lzmaProps.dictSize; 226 UInt64 blockSize = (UInt64)dictSize << 2; 227 const UInt32 kMinSize = (UInt32)1 << 20; 228 const UInt32 kMaxSize = (UInt32)1 << 28; 229 if (blockSize < kMinSize) blockSize = kMinSize; 230 if (blockSize > kMaxSize) blockSize = kMaxSize; 231 if (blockSize < dictSize) blockSize = dictSize; 232 p->blockSize = (size_t)blockSize; 233 } 234 } 235 236 static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) 237 { 238 return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; 239 } 240 241 /* ---------- Lzma2 ---------- */ 242 243 typedef struct 244 { 245 Byte propEncoded; 246 CLzma2EncProps props; 247 248 Byte *outBuf; 249 250 ISzAlloc *alloc; 251 ISzAlloc *allocBig; 252 253 CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX]; 254 255 #ifndef _7ZIP_ST 256 CMtCoder mtCoder; 257 #endif 258 259 } CLzma2Enc; 260 261 262 /* ---------- Lzma2EncThread ---------- */ 263 264 static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder, 265 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) 266 { 267 UInt64 packTotal = 0; 268 SRes res = SZ_OK; 269 270 if (mainEncoder->outBuf == 0) 271 { 272 mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); 273 if (mainEncoder->outBuf == 0) 274 return SZ_ERROR_MEM; 275 } 276 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); 277 RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE, 278 mainEncoder->alloc, mainEncoder->allocBig)); 279 for (;;) 280 { 281 size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; 282 res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream); 283 if (res != SZ_OK) 284 break; 285 packTotal += packSize; 286 res = Progress(progress, p->srcPos, packTotal); 287 if (res != SZ_OK) 288 break; 289 if (packSize == 0) 290 break; 291 } 292 LzmaEnc_Finish(p->enc); 293 if (res == SZ_OK) 294 { 295 Byte b = 0; 296 if (outStream->Write(outStream, &b, 1) != 1) 297 return SZ_ERROR_WRITE; 298 } 299 return res; 300 } 301 302 #ifndef _7ZIP_ST 303 304 typedef struct 305 { 306 IMtCoderCallback funcTable; 307 CLzma2Enc *lzma2Enc; 308 } CMtCallbackImp; 309 310 static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize, 311 const Byte *src, size_t srcSize, int finished) 312 { 313 CMtCallbackImp *imp = (CMtCallbackImp *)pp; 314 CLzma2Enc *mainEncoder = imp->lzma2Enc; 315 CLzma2EncInt *p = &mainEncoder->coders[index]; 316 317 SRes res = SZ_OK; 318 { 319 size_t destLim = *destSize; 320 *destSize = 0; 321 322 if (srcSize != 0) 323 { 324 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); 325 326 RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE, 327 mainEncoder->alloc, mainEncoder->allocBig)); 328 329 while (p->srcPos < srcSize) 330 { 331 size_t packSize = destLim - *destSize; 332 res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL); 333 if (res != SZ_OK) 334 break; 335 *destSize += packSize; 336 337 if (packSize == 0) 338 { 339 res = SZ_ERROR_FAIL; 340 break; 341 } 342 343 if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK) 344 { 345 res = SZ_ERROR_PROGRESS; 346 break; 347 } 348 } 349 LzmaEnc_Finish(p->enc); 350 if (res != SZ_OK) 351 return res; 352 } 353 if (finished) 354 { 355 if (*destSize == destLim) 356 return SZ_ERROR_OUTPUT_EOF; 357 dest[(*destSize)++] = 0; 358 } 359 } 360 return res; 361 } 362 363 #endif 364 365 /* ---------- Lzma2Enc ---------- */ 366 367 CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig) 368 { 369 CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc)); 370 if (p == 0) 371 return NULL; 372 Lzma2EncProps_Init(&p->props); 373 Lzma2EncProps_Normalize(&p->props); 374 p->outBuf = 0; 375 p->alloc = alloc; 376 p->allocBig = allocBig; 377 { 378 unsigned i; 379 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) 380 p->coders[i].enc = 0; 381 } 382 #ifndef _7ZIP_ST 383 MtCoder_Construct(&p->mtCoder); 384 #endif 385 386 return p; 387 } 388 389 void Lzma2Enc_Destroy(CLzma2EncHandle pp) 390 { 391 CLzma2Enc *p = (CLzma2Enc *)pp; 392 unsigned i; 393 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) 394 { 395 CLzma2EncInt *t = &p->coders[i]; 396 if (t->enc) 397 { 398 LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); 399 t->enc = 0; 400 } 401 } 402 403 #ifndef _7ZIP_ST 404 MtCoder_Destruct(&p->mtCoder); 405 #endif 406 407 IAlloc_Free(p->alloc, p->outBuf); 408 IAlloc_Free(p->alloc, pp); 409 } 410 411 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) 412 { 413 CLzma2Enc *p = (CLzma2Enc *)pp; 414 CLzmaEncProps lzmaProps = props->lzmaProps; 415 LzmaEncProps_Normalize(&lzmaProps); 416 if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX) 417 return SZ_ERROR_PARAM; 418 p->props = *props; 419 Lzma2EncProps_Normalize(&p->props); 420 return SZ_OK; 421 } 422 423 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) 424 { 425 CLzma2Enc *p = (CLzma2Enc *)pp; 426 unsigned i; 427 UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps); 428 for (i = 0; i < 40; i++) 429 if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i)) 430 break; 431 return (Byte)i; 432 } 433 434 SRes Lzma2Enc_Encode(CLzma2EncHandle pp, 435 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) 436 { 437 CLzma2Enc *p = (CLzma2Enc *)pp; 438 int i; 439 440 for (i = 0; i < p->props.numBlockThreads; i++) 441 { 442 CLzma2EncInt *t = &p->coders[i]; 443 if (t->enc == NULL) 444 { 445 t->enc = LzmaEnc_Create(p->alloc); 446 if (t->enc == NULL) 447 return SZ_ERROR_MEM; 448 } 449 } 450 451 #ifndef _7ZIP_ST 452 if (p->props.numBlockThreads <= 1) 453 #endif 454 return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress); 455 456 #ifndef _7ZIP_ST 457 458 { 459 CMtCallbackImp mtCallback; 460 461 mtCallback.funcTable.Code = MtCallbackImp_Code; 462 mtCallback.lzma2Enc = p; 463 464 p->mtCoder.progress = progress; 465 p->mtCoder.inStream = inStream; 466 p->mtCoder.outStream = outStream; 467 p->mtCoder.alloc = p->alloc; 468 p->mtCoder.mtCallback = &mtCallback.funcTable; 469 470 p->mtCoder.blockSize = p->props.blockSize; 471 p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16; 472 p->mtCoder.numThreads = p->props.numBlockThreads; 473 474 return MtCoder_Code(&p->mtCoder); 475 } 476 #endif 477 } 478
