1 /* Lzma2Enc.c -- LZMA2 Encoder 2 2015-10-04 : Igor Pavlov : Public domain */ 3 4 #include "Precomp.h" 5 6 /* #include <stdio.h> */ 7 #include <string.h> 8 9 /* #define _7ZIP_ST */ 10 11 #include "Lzma2Enc.h" 12 13 #ifndef _7ZIP_ST 14 #include "MtCoder.h" 15 #else 16 #define NUM_MT_CODER_THREADS_MAX 1 17 #endif 18 19 #define LZMA2_CONTROL_LZMA (1 << 7) 20 #define LZMA2_CONTROL_COPY_NO_RESET 2 21 #define LZMA2_CONTROL_COPY_RESET_DIC 1 22 #define LZMA2_CONTROL_EOF 0 23 24 #define LZMA2_LCLP_MAX 4 25 26 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) 27 28 #define LZMA2_PACK_SIZE_MAX (1 << 16) 29 #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX 30 #define LZMA2_UNPACK_SIZE_MAX (1 << 21) 31 #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX 32 33 #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16) 34 35 36 #define PRF(x) /* x */ 37 38 /* ---------- CLzma2EncInt ---------- */ 39 40 typedef struct 41 { 42 CLzmaEncHandle enc; 43 UInt64 srcPos; 44 Byte props; 45 Bool needInitState; 46 Bool needInitProp; 47 } CLzma2EncInt; 48 49 static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props) 50 { 51 Byte propsEncoded[LZMA_PROPS_SIZE]; 52 SizeT propsSize = LZMA_PROPS_SIZE; 53 RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); 54 RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); 55 p->srcPos = 0; 56 p->props = propsEncoded[0]; 57 p->needInitState = True; 58 p->needInitProp = True; 59 return SZ_OK; 60 } 61 62 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, 63 ISzAlloc *alloc, ISzAlloc *allocBig); 64 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, 65 UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig); 66 SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, 67 Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); 68 const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); 69 void LzmaEnc_Finish(CLzmaEncHandle pp); 70 void LzmaEnc_SaveState(CLzmaEncHandle pp); 71 void LzmaEnc_RestoreState(CLzmaEncHandle pp); 72 73 74 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, 75 size_t *packSizeRes, ISeqOutStream *outStream) 76 { 77 size_t packSizeLimit = *packSizeRes; 78 size_t packSize = packSizeLimit; 79 UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; 80 unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); 81 Bool useCopyBlock; 82 SRes res; 83 84 *packSizeRes = 0; 85 if (packSize < lzHeaderSize) 86 return SZ_ERROR_OUTPUT_EOF; 87 packSize -= lzHeaderSize; 88 89 LzmaEnc_SaveState(p->enc); 90 res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState, 91 outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize); 92 93 PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize)); 94 95 if (unpackSize == 0) 96 return res; 97 98 if (res == SZ_OK) 99 useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16)); 100 else 101 { 102 if (res != SZ_ERROR_OUTPUT_EOF) 103 return res; 104 res = SZ_OK; 105 useCopyBlock = True; 106 } 107 108 if (useCopyBlock) 109 { 110 size_t destPos = 0; 111 PRF(printf("################# COPY ")); 112 113 while (unpackSize > 0) 114 { 115 UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE; 116 if (packSizeLimit - destPos < u + 3) 117 return SZ_ERROR_OUTPUT_EOF; 118 outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET); 119 outBuf[destPos++] = (Byte)((u - 1) >> 8); 120 outBuf[destPos++] = (Byte)(u - 1); 121 memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u); 122 unpackSize -= u; 123 destPos += u; 124 p->srcPos += u; 125 126 if (outStream) 127 { 128 *packSizeRes += destPos; 129 if (outStream->Write(outStream, outBuf, destPos) != destPos) 130 return SZ_ERROR_WRITE; 131 destPos = 0; 132 } 133 else 134 *packSizeRes = destPos; 135 /* needInitState = True; */ 136 } 137 138 LzmaEnc_RestoreState(p->enc); 139 return SZ_OK; 140 } 141 142 { 143 size_t destPos = 0; 144 UInt32 u = unpackSize - 1; 145 UInt32 pm = (UInt32)(packSize - 1); 146 unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0); 147 148 PRF(printf(" ")); 149 150 outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F)); 151 outBuf[destPos++] = (Byte)(u >> 8); 152 outBuf[destPos++] = (Byte)u; 153 outBuf[destPos++] = (Byte)(pm >> 8); 154 outBuf[destPos++] = (Byte)pm; 155 156 if (p->needInitProp) 157 outBuf[destPos++] = p->props; 158 159 p->needInitProp = False; 160 p->needInitState = False; 161 destPos += packSize; 162 p->srcPos += unpackSize; 163 164 if (outStream) 165 if (outStream->Write(outStream, outBuf, destPos) != destPos) 166 return SZ_ERROR_WRITE; 167 168 *packSizeRes = destPos; 169 return SZ_OK; 170 } 171 } 172 173 174 /* ---------- Lzma2 Props ---------- */ 175 176 void Lzma2EncProps_Init(CLzma2EncProps *p) 177 { 178 LzmaEncProps_Init(&p->lzmaProps); 179 p->numTotalThreads = -1; 180 p->numBlockThreads = -1; 181 p->blockSize = 0; 182 } 183 184 void Lzma2EncProps_Normalize(CLzma2EncProps *p) 185 { 186 int t1, t1n, t2, t3; 187 { 188 CLzmaEncProps lzmaProps = p->lzmaProps; 189 LzmaEncProps_Normalize(&lzmaProps); 190 t1n = lzmaProps.numThreads; 191 } 192 193 t1 = p->lzmaProps.numThreads; 194 t2 = p->numBlockThreads; 195 t3 = p->numTotalThreads; 196 197 if (t2 > NUM_MT_CODER_THREADS_MAX) 198 t2 = NUM_MT_CODER_THREADS_MAX; 199 200 if (t3 <= 0) 201 { 202 if (t2 <= 0) 203 t2 = 1; 204 t3 = t1n * t2; 205 } 206 else if (t2 <= 0) 207 { 208 t2 = t3 / t1n; 209 if (t2 == 0) 210 { 211 t1 = 1; 212 t2 = t3; 213 } 214 if (t2 > NUM_MT_CODER_THREADS_MAX) 215 t2 = NUM_MT_CODER_THREADS_MAX; 216 } 217 else if (t1 <= 0) 218 { 219 t1 = t3 / t2; 220 if (t1 == 0) 221 t1 = 1; 222 } 223 else 224 t3 = t1n * t2; 225 226 p->lzmaProps.numThreads = t1; 227 228 LzmaEncProps_Normalize(&p->lzmaProps); 229 230 t1 = p->lzmaProps.numThreads; 231 232 if (p->blockSize == 0) 233 { 234 UInt32 dictSize = p->lzmaProps.dictSize; 235 UInt64 blockSize = (UInt64)dictSize << 2; 236 const UInt32 kMinSize = (UInt32)1 << 20; 237 const UInt32 kMaxSize = (UInt32)1 << 28; 238 if (blockSize < kMinSize) blockSize = kMinSize; 239 if (blockSize > kMaxSize) blockSize = kMaxSize; 240 if (blockSize < dictSize) blockSize = dictSize; 241 p->blockSize = (size_t)blockSize; 242 } 243 244 if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1) 245 { 246 UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1; 247 if (temp > p->lzmaProps.reduceSize) 248 { 249 UInt64 numBlocks = temp / p->blockSize; 250 if (numBlocks < (unsigned)t2) 251 { 252 t2 = (unsigned)numBlocks; 253 if (t2 == 0) 254 t2 = 1; 255 t3 = t1 * t2; 256 } 257 } 258 } 259 260 p->numBlockThreads = t2; 261 p->numTotalThreads = t3; 262 } 263 264 265 static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) 266 { 267 return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; 268 } 269 270 271 /* ---------- Lzma2 ---------- */ 272 273 typedef struct 274 { 275 Byte propEncoded; 276 CLzma2EncProps props; 277 278 Byte *outBuf; 279 280 ISzAlloc *alloc; 281 ISzAlloc *allocBig; 282 283 CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX]; 284 285 #ifndef _7ZIP_ST 286 CMtCoder mtCoder; 287 #endif 288 289 } CLzma2Enc; 290 291 292 /* ---------- Lzma2EncThread ---------- */ 293 294 static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder, 295 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) 296 { 297 UInt64 packTotal = 0; 298 SRes res = SZ_OK; 299 300 if (!mainEncoder->outBuf) 301 { 302 mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); 303 if (!mainEncoder->outBuf) 304 return SZ_ERROR_MEM; 305 } 306 307 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); 308 RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE, 309 mainEncoder->alloc, mainEncoder->allocBig)); 310 311 for (;;) 312 { 313 size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; 314 res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream); 315 if (res != SZ_OK) 316 break; 317 packTotal += packSize; 318 res = Progress(progress, p->srcPos, packTotal); 319 if (res != SZ_OK) 320 break; 321 if (packSize == 0) 322 break; 323 } 324 325 LzmaEnc_Finish(p->enc); 326 327 if (res == SZ_OK) 328 { 329 Byte b = 0; 330 if (outStream->Write(outStream, &b, 1) != 1) 331 return SZ_ERROR_WRITE; 332 } 333 334 return res; 335 } 336 337 338 #ifndef _7ZIP_ST 339 340 typedef struct 341 { 342 IMtCoderCallback funcTable; 343 CLzma2Enc *lzma2Enc; 344 } CMtCallbackImp; 345 346 static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize, 347 const Byte *src, size_t srcSize, int finished) 348 { 349 CMtCallbackImp *imp = (CMtCallbackImp *)pp; 350 CLzma2Enc *mainEncoder = imp->lzma2Enc; 351 CLzma2EncInt *p = &mainEncoder->coders[index]; 352 353 SRes res = SZ_OK; 354 { 355 size_t destLim = *destSize; 356 *destSize = 0; 357 358 if (srcSize != 0) 359 { 360 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); 361 362 RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE, 363 mainEncoder->alloc, mainEncoder->allocBig)); 364 365 while (p->srcPos < srcSize) 366 { 367 size_t packSize = destLim - *destSize; 368 res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL); 369 if (res != SZ_OK) 370 break; 371 *destSize += packSize; 372 373 if (packSize == 0) 374 { 375 res = SZ_ERROR_FAIL; 376 break; 377 } 378 379 if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK) 380 { 381 res = SZ_ERROR_PROGRESS; 382 break; 383 } 384 } 385 386 LzmaEnc_Finish(p->enc); 387 if (res != SZ_OK) 388 return res; 389 } 390 391 if (finished) 392 { 393 if (*destSize == destLim) 394 return SZ_ERROR_OUTPUT_EOF; 395 dest[(*destSize)++] = 0; 396 } 397 } 398 return res; 399 } 400 401 #endif 402 403 404 /* ---------- Lzma2Enc ---------- */ 405 406 CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig) 407 { 408 CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc)); 409 if (!p) 410 return NULL; 411 Lzma2EncProps_Init(&p->props); 412 Lzma2EncProps_Normalize(&p->props); 413 p->outBuf = 0; 414 p->alloc = alloc; 415 p->allocBig = allocBig; 416 { 417 unsigned i; 418 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) 419 p->coders[i].enc = 0; 420 } 421 422 #ifndef _7ZIP_ST 423 MtCoder_Construct(&p->mtCoder); 424 #endif 425 426 return p; 427 } 428 429 void Lzma2Enc_Destroy(CLzma2EncHandle pp) 430 { 431 CLzma2Enc *p = (CLzma2Enc *)pp; 432 unsigned i; 433 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) 434 { 435 CLzma2EncInt *t = &p->coders[i]; 436 if (t->enc) 437 { 438 LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); 439 t->enc = 0; 440 } 441 } 442 443 #ifndef _7ZIP_ST 444 MtCoder_Destruct(&p->mtCoder); 445 #endif 446 447 IAlloc_Free(p->alloc, p->outBuf); 448 IAlloc_Free(p->alloc, pp); 449 } 450 451 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) 452 { 453 CLzma2Enc *p = (CLzma2Enc *)pp; 454 CLzmaEncProps lzmaProps = props->lzmaProps; 455 LzmaEncProps_Normalize(&lzmaProps); 456 if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX) 457 return SZ_ERROR_PARAM; 458 p->props = *props; 459 Lzma2EncProps_Normalize(&p->props); 460 return SZ_OK; 461 } 462 463 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) 464 { 465 CLzma2Enc *p = (CLzma2Enc *)pp; 466 unsigned i; 467 UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps); 468 for (i = 0; i < 40; i++) 469 if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i)) 470 break; 471 return (Byte)i; 472 } 473 474 SRes Lzma2Enc_Encode(CLzma2EncHandle pp, 475 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) 476 { 477 CLzma2Enc *p = (CLzma2Enc *)pp; 478 int i; 479 480 for (i = 0; i < p->props.numBlockThreads; i++) 481 { 482 CLzma2EncInt *t = &p->coders[(unsigned)i]; 483 if (!t->enc) 484 { 485 t->enc = LzmaEnc_Create(p->alloc); 486 if (!t->enc) 487 return SZ_ERROR_MEM; 488 } 489 } 490 491 #ifndef _7ZIP_ST 492 if (p->props.numBlockThreads > 1) 493 { 494 CMtCallbackImp mtCallback; 495 496 mtCallback.funcTable.Code = MtCallbackImp_Code; 497 mtCallback.lzma2Enc = p; 498 499 p->mtCoder.progress = progress; 500 p->mtCoder.inStream = inStream; 501 p->mtCoder.outStream = outStream; 502 p->mtCoder.alloc = p->alloc; 503 p->mtCoder.mtCallback = &mtCallback.funcTable; 504 505 p->mtCoder.blockSize = p->props.blockSize; 506 p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16; 507 if (p->mtCoder.destBlockSize < p->props.blockSize) 508 { 509 p->mtCoder.destBlockSize = (size_t)0 - 1; 510 if (p->mtCoder.destBlockSize < p->props.blockSize) 511 return SZ_ERROR_FAIL; 512 } 513 p->mtCoder.numThreads = p->props.numBlockThreads; 514 515 return MtCoder_Code(&p->mtCoder); 516 } 517 #endif 518 519 return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress); 520 } 521
