1 /* Lzma2Enc.c -- LZMA2 Encoder 2 2012-06-19 : Igor Pavlov : Public domain */ 3 4 #include "Precomp.h" 5 6 /* #include <stdio.h> */ 7 #include <string.h> 8 9 /* #define _7ZIP_ST */ 10 11 #include "Lzma2Enc.h" 12 13 #ifndef _7ZIP_ST 14 #include "MtCoder.h" 15 #else 16 #define NUM_MT_CODER_THREADS_MAX 1 17 #endif 18 19 #define LZMA2_CONTROL_LZMA (1 << 7) 20 #define LZMA2_CONTROL_COPY_NO_RESET 2 21 #define LZMA2_CONTROL_COPY_RESET_DIC 1 22 #define LZMA2_CONTROL_EOF 0 23 24 #define LZMA2_LCLP_MAX 4 25 26 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) 27 28 #define LZMA2_PACK_SIZE_MAX (1 << 16) 29 #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX 30 #define LZMA2_UNPACK_SIZE_MAX (1 << 21) 31 #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX 32 33 #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16) 34 35 36 #define PRF(x) /* x */ 37 38 /* ---------- CLzma2EncInt ---------- */ 39 40 typedef struct 41 { 42 CLzmaEncHandle enc; 43 UInt64 srcPos; 44 Byte props; 45 Bool needInitState; 46 Bool needInitProp; 47 } CLzma2EncInt; 48 49 static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props) 50 { 51 Byte propsEncoded[LZMA_PROPS_SIZE]; 52 SizeT propsSize = LZMA_PROPS_SIZE; 53 RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); 54 RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); 55 p->srcPos = 0; 56 p->props = propsEncoded[0]; 57 p->needInitState = True; 58 p->needInitProp = True; 59 return SZ_OK; 60 } 61 62 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, 63 ISzAlloc *alloc, ISzAlloc *allocBig); 64 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, 65 UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig); 66 SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, 67 Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); 68 const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); 69 void LzmaEnc_Finish(CLzmaEncHandle pp); 70 void LzmaEnc_SaveState(CLzmaEncHandle pp); 71 void LzmaEnc_RestoreState(CLzmaEncHandle pp); 72 73 74 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, 75 size_t *packSizeRes, ISeqOutStream *outStream) 76 { 77 size_t packSizeLimit = *packSizeRes; 78 size_t packSize = packSizeLimit; 79 UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; 80 unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); 81 Bool useCopyBlock; 82 SRes res; 83 84 *packSizeRes = 0; 85 if (packSize < lzHeaderSize) 86 return SZ_ERROR_OUTPUT_EOF; 87 packSize -= lzHeaderSize; 88 89 LzmaEnc_SaveState(p->enc); 90 res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState, 91 outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize); 92 93 PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize)); 94 95 if (unpackSize == 0) 96 return res; 97 98 if (res == SZ_OK) 99 useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16)); 100 else 101 { 102 if (res != SZ_ERROR_OUTPUT_EOF) 103 return res; 104 res = SZ_OK; 105 useCopyBlock = True; 106 } 107 108 if (useCopyBlock) 109 { 110 size_t destPos = 0; 111 PRF(printf("################# COPY ")); 112 while (unpackSize > 0) 113 { 114 UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE; 115 if (packSizeLimit - destPos < u + 3) 116 return SZ_ERROR_OUTPUT_EOF; 117 outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET); 118 outBuf[destPos++] = (Byte)((u - 1) >> 8); 119 outBuf[destPos++] = (Byte)(u - 1); 120 memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u); 121 unpackSize -= u; 122 destPos += u; 123 p->srcPos += u; 124 if (outStream) 125 { 126 *packSizeRes += destPos; 127 if (outStream->Write(outStream, outBuf, destPos) != destPos) 128 return SZ_ERROR_WRITE; 129 destPos = 0; 130 } 131 else 132 *packSizeRes = destPos; 133 /* needInitState = True; */ 134 } 135 LzmaEnc_RestoreState(p->enc); 136 return SZ_OK; 137 } 138 { 139 size_t destPos = 0; 140 UInt32 u = unpackSize - 1; 141 UInt32 pm = (UInt32)(packSize - 1); 142 unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0); 143 144 PRF(printf(" ")); 145 146 outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F)); 147 outBuf[destPos++] = (Byte)(u >> 8); 148 outBuf[destPos++] = (Byte)u; 149 outBuf[destPos++] = (Byte)(pm >> 8); 150 outBuf[destPos++] = (Byte)pm; 151 152 if (p->needInitProp) 153 outBuf[destPos++] = p->props; 154 155 p->needInitProp = False; 156 p->needInitState = False; 157 destPos += packSize; 158 p->srcPos += unpackSize; 159 160 if (outStream) 161 if (outStream->Write(outStream, outBuf, destPos) != destPos) 162 return SZ_ERROR_WRITE; 163 *packSizeRes = destPos; 164 return SZ_OK; 165 } 166 } 167 168 /* ---------- Lzma2 Props ---------- */ 169 170 void Lzma2EncProps_Init(CLzma2EncProps *p) 171 { 172 LzmaEncProps_Init(&p->lzmaProps); 173 p->numTotalThreads = -1; 174 p->numBlockThreads = -1; 175 p->blockSize = 0; 176 } 177 178 void Lzma2EncProps_Normalize(CLzma2EncProps *p) 179 { 180 int t1, t1n, t2, t3; 181 { 182 CLzmaEncProps lzmaProps = p->lzmaProps; 183 LzmaEncProps_Normalize(&lzmaProps); 184 t1n = lzmaProps.numThreads; 185 } 186 187 t1 = p->lzmaProps.numThreads; 188 t2 = p->numBlockThreads; 189 t3 = p->numTotalThreads; 190 191 if (t2 > NUM_MT_CODER_THREADS_MAX) 192 t2 = NUM_MT_CODER_THREADS_MAX; 193 194 if (t3 <= 0) 195 { 196 if (t2 <= 0) 197 t2 = 1; 198 t3 = t1n * t2; 199 } 200 else if (t2 <= 0) 201 { 202 t2 = t3 / t1n; 203 if (t2 == 0) 204 { 205 t1 = 1; 206 t2 = t3; 207 } 208 if (t2 > NUM_MT_CODER_THREADS_MAX) 209 t2 = NUM_MT_CODER_THREADS_MAX; 210 } 211 else if (t1 <= 0) 212 { 213 t1 = t3 / t2; 214 if (t1 == 0) 215 t1 = 1; 216 } 217 else 218 t3 = t1n * t2; 219 220 p->lzmaProps.numThreads = t1; 221 222 LzmaEncProps_Normalize(&p->lzmaProps); 223 224 if (p->blockSize == 0) 225 { 226 UInt32 dictSize = p->lzmaProps.dictSize; 227 UInt64 blockSize = (UInt64)dictSize << 2; 228 const UInt32 kMinSize = (UInt32)1 << 20; 229 const UInt32 kMaxSize = (UInt32)1 << 28; 230 if (blockSize < kMinSize) blockSize = kMinSize; 231 if (blockSize > kMaxSize) blockSize = kMaxSize; 232 if (blockSize < dictSize) blockSize = dictSize; 233 p->blockSize = (size_t)blockSize; 234 } 235 if (t2 > 1) 236 { 237 UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1; 238 if (temp > p->lzmaProps.reduceSize) 239 { 240 UInt64 numBlocks = temp / p->blockSize; 241 if (t2 >= 0 && numBlocks < (UInt64) t2) 242 { 243 t2 = (UInt32)numBlocks; 244 t3 = t1 * t2; 245 } 246 } 247 } 248 p->numBlockThreads = t2; 249 p->numTotalThreads = t3; 250 } 251 252 static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) 253 { 254 return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; 255 } 256 257 /* ---------- Lzma2 ---------- */ 258 259 typedef struct 260 { 261 Byte propEncoded; 262 CLzma2EncProps props; 263 264 Byte *outBuf; 265 266 ISzAlloc *alloc; 267 ISzAlloc *allocBig; 268 269 CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX]; 270 271 #ifndef _7ZIP_ST 272 CMtCoder mtCoder; 273 #endif 274 275 } CLzma2Enc; 276 277 278 /* ---------- Lzma2EncThread ---------- */ 279 280 static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder, 281 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) 282 { 283 UInt64 packTotal = 0; 284 SRes res = SZ_OK; 285 286 if (mainEncoder->outBuf == 0) 287 { 288 mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); 289 if (mainEncoder->outBuf == 0) 290 return SZ_ERROR_MEM; 291 } 292 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); 293 RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE, 294 mainEncoder->alloc, mainEncoder->allocBig)); 295 for (;;) 296 { 297 size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; 298 res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream); 299 if (res != SZ_OK) 300 break; 301 packTotal += packSize; 302 res = Progress(progress, p->srcPos, packTotal); 303 if (res != SZ_OK) 304 break; 305 if (packSize == 0) 306 break; 307 } 308 LzmaEnc_Finish(p->enc); 309 if (res == SZ_OK) 310 { 311 Byte b = 0; 312 if (outStream->Write(outStream, &b, 1) != 1) 313 return SZ_ERROR_WRITE; 314 } 315 return res; 316 } 317 318 #ifndef _7ZIP_ST 319 320 typedef struct 321 { 322 IMtCoderCallback funcTable; 323 CLzma2Enc *lzma2Enc; 324 } CMtCallbackImp; 325 326 static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize, 327 const Byte *src, size_t srcSize, int finished) 328 { 329 CMtCallbackImp *imp = (CMtCallbackImp *)pp; 330 CLzma2Enc *mainEncoder = imp->lzma2Enc; 331 CLzma2EncInt *p = &mainEncoder->coders[index]; 332 333 SRes res = SZ_OK; 334 { 335 size_t destLim = *destSize; 336 *destSize = 0; 337 338 if (srcSize != 0) 339 { 340 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); 341 342 RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE, 343 mainEncoder->alloc, mainEncoder->allocBig)); 344 345 while (p->srcPos < srcSize) 346 { 347 size_t packSize = destLim - *destSize; 348 res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL); 349 if (res != SZ_OK) 350 break; 351 *destSize += packSize; 352 353 if (packSize == 0) 354 { 355 res = SZ_ERROR_FAIL; 356 break; 357 } 358 359 if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK) 360 { 361 res = SZ_ERROR_PROGRESS; 362 break; 363 } 364 } 365 LzmaEnc_Finish(p->enc); 366 if (res != SZ_OK) 367 return res; 368 } 369 if (finished) 370 { 371 if (*destSize == destLim) 372 return SZ_ERROR_OUTPUT_EOF; 373 dest[(*destSize)++] = 0; 374 } 375 } 376 return res; 377 } 378 379 #endif 380 381 /* ---------- Lzma2Enc ---------- */ 382 383 CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig) 384 { 385 CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc)); 386 if (p == 0) 387 return NULL; 388 Lzma2EncProps_Init(&p->props); 389 Lzma2EncProps_Normalize(&p->props); 390 p->outBuf = 0; 391 p->alloc = alloc; 392 p->allocBig = allocBig; 393 { 394 unsigned i; 395 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) 396 p->coders[i].enc = 0; 397 } 398 #ifndef _7ZIP_ST 399 MtCoder_Construct(&p->mtCoder); 400 #endif 401 402 return p; 403 } 404 405 void Lzma2Enc_Destroy(CLzma2EncHandle pp) 406 { 407 CLzma2Enc *p = (CLzma2Enc *)pp; 408 unsigned i; 409 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) 410 { 411 CLzma2EncInt *t = &p->coders[i]; 412 if (t->enc) 413 { 414 LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); 415 t->enc = 0; 416 } 417 } 418 419 #ifndef _7ZIP_ST 420 MtCoder_Destruct(&p->mtCoder); 421 #endif 422 423 IAlloc_Free(p->alloc, p->outBuf); 424 IAlloc_Free(p->alloc, pp); 425 } 426 427 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) 428 { 429 CLzma2Enc *p = (CLzma2Enc *)pp; 430 CLzmaEncProps lzmaProps = props->lzmaProps; 431 LzmaEncProps_Normalize(&lzmaProps); 432 if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX) 433 return SZ_ERROR_PARAM; 434 p->props = *props; 435 Lzma2EncProps_Normalize(&p->props); 436 return SZ_OK; 437 } 438 439 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) 440 { 441 CLzma2Enc *p = (CLzma2Enc *)pp; 442 unsigned i; 443 UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps); 444 for (i = 0; i < 40; i++) 445 if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i)) 446 break; 447 return (Byte)i; 448 } 449 450 SRes Lzma2Enc_Encode(CLzma2EncHandle pp, 451 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) 452 { 453 CLzma2Enc *p = (CLzma2Enc *)pp; 454 int i; 455 456 for (i = 0; i < p->props.numBlockThreads; i++) 457 { 458 CLzma2EncInt *t = &p->coders[i]; 459 if (t->enc == NULL) 460 { 461 t->enc = LzmaEnc_Create(p->alloc); 462 if (t->enc == NULL) 463 return SZ_ERROR_MEM; 464 } 465 } 466 467 #ifndef _7ZIP_ST 468 if (p->props.numBlockThreads <= 1) 469 #endif 470 return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress); 471 472 #ifndef _7ZIP_ST 473 474 { 475 CMtCallbackImp mtCallback; 476 477 mtCallback.funcTable.Code = MtCallbackImp_Code; 478 mtCallback.lzma2Enc = p; 479 480 p->mtCoder.progress = progress; 481 p->mtCoder.inStream = inStream; 482 p->mtCoder.outStream = outStream; 483 p->mtCoder.alloc = p->alloc; 484 p->mtCoder.mtCallback = &mtCallback.funcTable; 485 486 p->mtCoder.blockSize = p->props.blockSize; 487 p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16; 488 p->mtCoder.numThreads = p->props.numBlockThreads; 489 490 return MtCoder_Code(&p->mtCoder); 491 } 492 #endif 493 } 494
