1 /* XzEnc.c -- Xz Encode 2 2014-12-30 : Igor Pavlov : Public domain */ 3 4 #include "Precomp.h" 5 6 #include <stdlib.h> 7 #include <string.h> 8 9 #include "7zCrc.h" 10 #include "Alloc.h" 11 #include "Bra.h" 12 #include "CpuArch.h" 13 #ifdef USE_SUBBLOCK 14 #include "Bcj3Enc.c" 15 #include "SbFind.c" 16 #include "SbEnc.c" 17 #endif 18 19 #include "XzEnc.h" 20 21 static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); } 22 static void SzBigFree(void *p, void *address) { p = p; BigFree(address); } 23 static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; 24 25 static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); } 26 static void SzFree(void *p, void *address) { p = p; MyFree(address); } 27 static ISzAlloc g_Alloc = { SzAlloc, SzFree }; 28 29 #define XzBlock_ClearFlags(p) (p)->flags = 0; 30 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); 31 #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; 32 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; 33 34 static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size) 35 { 36 return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; 37 } 38 39 static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc) 40 { 41 *crc = CrcUpdate(*crc, buf, size); 42 return WriteBytes(s, buf, size); 43 } 44 45 SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) 46 { 47 UInt32 crc; 48 Byte header[XZ_STREAM_HEADER_SIZE]; 49 memcpy(header, XZ_SIG, XZ_SIG_SIZE); 50 header[XZ_SIG_SIZE] = (Byte)(f >> 8); 51 header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF); 52 crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE); 53 SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc); 54 return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE); 55 } 56 57 SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s) 58 { 59 Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; 60 61 unsigned pos = 1; 62 int numFilters, i; 63 header[pos++] = p->flags; 64 65 if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize); 66 if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize); 67 numFilters = XzBlock_GetNumFilters(p); 68 for (i = 0; i < numFilters; i++) 69 { 70 const CXzFilter *f = &p->filters[i]; 71 pos += Xz_WriteVarInt(header + pos, f->id); 72 pos += Xz_WriteVarInt(header + pos, f->propsSize); 73 memcpy(header + pos, f->props, f->propsSize); 74 pos += f->propsSize; 75 } 76 while((pos & 3) != 0) 77 header[pos++] = 0; 78 header[0] = (Byte)(pos >> 2); 79 SetUi32(header + pos, CrcCalc(header, pos)); 80 return WriteBytes(s, header, pos + 4); 81 } 82 83 SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s) 84 { 85 Byte buf[32]; 86 UInt64 globalPos; 87 { 88 UInt32 crc = CRC_INIT_VAL; 89 unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); 90 size_t i; 91 92 globalPos = pos; 93 buf[0] = 0; 94 RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); 95 for (i = 0; i < p->numBlocks; i++) 96 { 97 const CXzBlockSizes *block = &p->blocks[i]; 98 pos = Xz_WriteVarInt(buf, block->totalSize); 99 pos += Xz_WriteVarInt(buf + pos, block->unpackSize); 100 globalPos += pos; 101 RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); 102 } 103 pos = ((unsigned)globalPos & 3); 104 if (pos != 0) 105 { 106 buf[0] = buf[1] = buf[2] = 0; 107 RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc)); 108 globalPos += 4 - pos; 109 } 110 { 111 SetUi32(buf, CRC_GET_DIGEST(crc)); 112 RINOK(WriteBytes(s, buf, 4)); 113 globalPos += 4; 114 } 115 } 116 117 { 118 UInt32 indexSize = (UInt32)((globalPos >> 2) - 1); 119 SetUi32(buf + 4, indexSize); 120 buf[8] = (Byte)(p->flags >> 8); 121 buf[9] = (Byte)(p->flags & 0xFF); 122 SetUi32(buf, CrcCalc(buf + 4, 6)); 123 memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE); 124 return WriteBytes(s, buf, 12); 125 } 126 } 127 128 SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc) 129 { 130 if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks) 131 { 132 size_t num = (p->numBlocks + 1) * 2; 133 size_t newSize = sizeof(CXzBlockSizes) * num; 134 CXzBlockSizes *blocks; 135 if (newSize / sizeof(CXzBlockSizes) != num) 136 return SZ_ERROR_MEM; 137 blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize); 138 if (blocks == 0) 139 return SZ_ERROR_MEM; 140 if (p->numBlocks != 0) 141 { 142 memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes)); 143 Xz_Free(p, alloc); 144 } 145 p->blocks = blocks; 146 p->numBlocksAllocated = num; 147 } 148 { 149 CXzBlockSizes *block = &p->blocks[p->numBlocks++]; 150 block->totalSize = totalSize; 151 block->unpackSize = unpackSize; 152 } 153 return SZ_OK; 154 } 155 156 /* ---------- CSeqCheckInStream ---------- */ 157 158 typedef struct 159 { 160 ISeqInStream p; 161 ISeqInStream *realStream; 162 UInt64 processed; 163 CXzCheck check; 164 } CSeqCheckInStream; 165 166 void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode) 167 { 168 p->processed = 0; 169 XzCheck_Init(&p->check, mode); 170 } 171 172 void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) 173 { 174 XzCheck_Final(&p->check, digest); 175 } 176 177 static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size) 178 { 179 CSeqCheckInStream *p = (CSeqCheckInStream *)pp; 180 SRes res = p->realStream->Read(p->realStream, data, size); 181 XzCheck_Update(&p->check, data, *size); 182 p->processed += *size; 183 return res; 184 } 185 186 /* ---------- CSeqSizeOutStream ---------- */ 187 188 typedef struct 189 { 190 ISeqOutStream p; 191 ISeqOutStream *realStream; 192 UInt64 processed; 193 } CSeqSizeOutStream; 194 195 static size_t MyWrite(void *pp, const void *data, size_t size) 196 { 197 CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp; 198 size = p->realStream->Write(p->realStream, data, size); 199 p->processed += size; 200 return size; 201 } 202 203 /* ---------- CSeqInFilter ---------- */ 204 205 #define FILTER_BUF_SIZE (1 << 20) 206 207 typedef struct 208 { 209 ISeqInStream p; 210 ISeqInStream *realStream; 211 IStateCoder StateCoder; 212 Byte *buf; 213 size_t curPos; 214 size_t endPos; 215 int srcWasFinished; 216 } CSeqInFilter; 217 218 static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) 219 { 220 CSeqInFilter *p = (CSeqInFilter *)pp; 221 size_t sizeOriginal = *size; 222 if (sizeOriginal == 0) 223 return SZ_OK; 224 *size = 0; 225 for (;;) 226 { 227 if (!p->srcWasFinished && p->curPos == p->endPos) 228 { 229 p->curPos = 0; 230 p->endPos = FILTER_BUF_SIZE; 231 RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos)); 232 if (p->endPos == 0) 233 p->srcWasFinished = 1; 234 } 235 { 236 SizeT srcLen = p->endPos - p->curPos; 237 int wasFinished; 238 SRes res; 239 *size = sizeOriginal; 240 res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen, 241 p->srcWasFinished, CODER_FINISH_ANY, &wasFinished); 242 p->curPos += srcLen; 243 if (*size != 0 || srcLen == 0 || res != 0) 244 return res; 245 } 246 } 247 } 248 249 static void SeqInFilter_Construct(CSeqInFilter *p) 250 { 251 p->buf = NULL; 252 p->p.Read = SeqInFilter_Read; 253 } 254 255 static void SeqInFilter_Free(CSeqInFilter *p) 256 { 257 if (p->buf) 258 { 259 g_Alloc.Free(&g_Alloc, p->buf); 260 p->buf = NULL; 261 } 262 } 263 264 SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc); 265 266 static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props) 267 { 268 if (!p->buf) 269 { 270 p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE); 271 if (!p->buf) 272 return SZ_ERROR_MEM; 273 } 274 p->curPos = p->endPos = 0; 275 p->srcWasFinished = 0; 276 RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc)); 277 RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc)); 278 p->StateCoder.Init(p->StateCoder.p); 279 return SZ_OK; 280 } 281 282 /* ---------- CSbEncInStream ---------- */ 283 284 #ifdef USE_SUBBLOCK 285 286 typedef struct 287 { 288 ISeqInStream p; 289 ISeqInStream *inStream; 290 CSbEnc enc; 291 } CSbEncInStream; 292 293 static SRes SbEncInStream_Read(void *pp, void *data, size_t *size) 294 { 295 CSbEncInStream *p = (CSbEncInStream *)pp; 296 size_t sizeOriginal = *size; 297 if (sizeOriginal == 0) 298 return S_OK; 299 for (;;) 300 { 301 if (p->enc.needRead && !p->enc.readWasFinished) 302 { 303 size_t processed = p->enc.needReadSizeMax; 304 RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed)); 305 p->enc.readPos += processed; 306 if (processed == 0) 307 { 308 p->enc.readWasFinished = True; 309 p->enc.isFinalFinished = True; 310 } 311 p->enc.needRead = False; 312 } 313 *size = sizeOriginal; 314 RINOK(SbEnc_Read(&p->enc, data, size)); 315 if (*size != 0 || !p->enc.needRead) 316 return S_OK; 317 } 318 } 319 320 void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc) 321 { 322 SbEnc_Construct(&p->enc, alloc); 323 p->p.Read = SbEncInStream_Read; 324 } 325 326 SRes SbEncInStream_Init(CSbEncInStream *p) 327 { 328 return SbEnc_Init(&p->enc); 329 } 330 331 void SbEncInStream_Free(CSbEncInStream *p) 332 { 333 SbEnc_Free(&p->enc); 334 } 335 336 #endif 337 338 339 typedef struct 340 { 341 CLzma2EncHandle lzma2; 342 #ifdef USE_SUBBLOCK 343 CSbEncInStream sb; 344 #endif 345 CSeqInFilter filter; 346 ISzAlloc *alloc; 347 ISzAlloc *bigAlloc; 348 } CLzma2WithFilters; 349 350 351 static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc) 352 { 353 p->alloc = alloc; 354 p->bigAlloc = bigAlloc; 355 p->lzma2 = NULL; 356 #ifdef USE_SUBBLOCK 357 SbEncInStream_Construct(&p->sb, alloc); 358 #endif 359 SeqInFilter_Construct(&p->filter); 360 } 361 362 static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p) 363 { 364 p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc); 365 if (p->lzma2 == 0) 366 return SZ_ERROR_MEM; 367 return SZ_OK; 368 } 369 370 static void Lzma2WithFilters_Free(CLzma2WithFilters *p) 371 { 372 SeqInFilter_Free(&p->filter); 373 #ifdef USE_SUBBLOCK 374 SbEncInStream_Free(&p->sb); 375 #endif 376 if (p->lzma2) 377 { 378 Lzma2Enc_Destroy(p->lzma2); 379 p->lzma2 = NULL; 380 } 381 } 382 383 void XzProps_Init(CXzProps *p) 384 { 385 p->lzma2Props = 0; 386 p->filterProps = 0; 387 p->checkId = XZ_CHECK_CRC32; 388 } 389 390 void XzFilterProps_Init(CXzFilterProps *p) 391 { 392 p->id = 0; 393 p->delta = 0; 394 p->ip= 0; 395 p->ipDefined = False; 396 } 397 398 static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf, 399 ISeqOutStream *outStream, ISeqInStream *inStream, 400 const CXzProps *props, ICompressProgress *progress) 401 { 402 xz->flags = (Byte)props->checkId; 403 404 RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props)); 405 RINOK(Xz_WriteHeader(xz->flags, outStream)); 406 407 { 408 CSeqCheckInStream checkInStream; 409 CSeqSizeOutStream seqSizeOutStream; 410 CXzBlock block; 411 int filterIndex = 0; 412 CXzFilter *filter = NULL; 413 const CXzFilterProps *fp = props->filterProps; 414 415 XzBlock_ClearFlags(&block); 416 XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); 417 418 if (fp) 419 { 420 filter = &block.filters[filterIndex++]; 421 filter->id = fp->id; 422 filter->propsSize = 0; 423 if (fp->id == XZ_ID_Delta) 424 { 425 filter->props[0] = (Byte)(fp->delta - 1); 426 filter->propsSize = 1; 427 } 428 else if (fp->ipDefined) 429 { 430 SetUi32(filter->props, fp->ip); 431 filter->propsSize = 4; 432 } 433 } 434 435 { 436 CXzFilter *f = &block.filters[filterIndex++]; 437 f->id = XZ_ID_LZMA2; 438 f->propsSize = 1; 439 f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); 440 } 441 442 seqSizeOutStream.p.Write = MyWrite; 443 seqSizeOutStream.realStream = outStream; 444 seqSizeOutStream.processed = 0; 445 446 RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p)); 447 448 checkInStream.p.Read = SeqCheckInStream_Read; 449 checkInStream.realStream = inStream; 450 SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags)); 451 452 if (fp) 453 { 454 #ifdef USE_SUBBLOCK 455 if (fp->id == XZ_ID_Subblock) 456 { 457 lzmaf->sb.inStream = &checkInStream.p; 458 RINOK(SbEncInStream_Init(&lzmaf->sb)); 459 } 460 else 461 #endif 462 { 463 lzmaf->filter.realStream = &checkInStream.p; 464 RINOK(SeqInFilter_Init(&lzmaf->filter, filter)); 465 } 466 } 467 468 { 469 UInt64 packPos = seqSizeOutStream.processed; 470 SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p, 471 fp ? 472 #ifdef USE_SUBBLOCK 473 (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p: 474 #endif 475 &lzmaf->filter.p: 476 &checkInStream.p, 477 progress); 478 RINOK(res); 479 block.unpackSize = checkInStream.processed; 480 block.packSize = seqSizeOutStream.processed - packPos; 481 } 482 483 { 484 unsigned padSize = 0; 485 Byte buf[128]; 486 while((((unsigned)block.packSize + padSize) & 3) != 0) 487 buf[padSize++] = 0; 488 SeqCheckInStream_GetDigest(&checkInStream, buf + padSize); 489 RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags))); 490 RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc)); 491 } 492 } 493 return Xz_WriteFooter(xz, outStream); 494 } 495 496 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, 497 const CXzProps *props, ICompressProgress *progress) 498 { 499 SRes res; 500 CXzStream xz; 501 CLzma2WithFilters lzmaf; 502 Xz_Construct(&xz); 503 Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc); 504 res = Lzma2WithFilters_Create(&lzmaf); 505 if (res == SZ_OK) 506 res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress); 507 Lzma2WithFilters_Free(&lzmaf); 508 Xz_Free(&xz, &g_Alloc); 509 return res; 510 } 511 512 SRes Xz_EncodeEmpty(ISeqOutStream *outStream) 513 { 514 SRes res; 515 CXzStream xz; 516 Xz_Construct(&xz); 517 res = Xz_WriteHeader(xz.flags, outStream); 518 if (res == SZ_OK) 519 res = Xz_WriteFooter(&xz, outStream); 520 Xz_Free(&xz, &g_Alloc); 521 return res; 522 } 523