Home | History | Annotate | Download | only in C 
      1 /* XzEnc.c -- Xz Encode
      2 2009-06-04 : Igor Pavlov : Public domain */
      3 
      4 #include <stdlib.h>
      5 #include <string.h>
      6 
      7 #include "7zCrc.h"
      8 #include "Alloc.h"
      9 #include "Bra.h"
     10 #include "CpuArch.h"
     11 #ifdef USE_SUBBLOCK
     12 #include "SbEnc.h"
     13 #endif
     14 
     15 #include "XzEnc.h"
     16 
     17 static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); }
     18 static void SzBigFree(void *p, void *address) { p = p; BigFree(address); }
     19 static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
     20 
     21 static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
     22 static void SzFree(void *p, void *address) { p = p; MyFree(address); }
     23 static ISzAlloc g_Alloc = { SzAlloc, SzFree };
     24 
     25 #define XzBlock_ClearFlags(p)       (p)->flags = 0;
     26 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
     27 #define XzBlock_SetHasPackSize(p)   (p)->flags |= XZ_BF_PACK_SIZE;
     28 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
     29 
     30 static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
     31 {
     32   return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
     33 }
     34 
     35 static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
     36 {
     37   *crc = CrcUpdate(*crc, buf, size);
     38   return WriteBytes(s, buf, size);
     39 }
     40 
     41 SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
     42 {
     43   UInt32 crc;
     44   Byte header[XZ_STREAM_HEADER_SIZE];
     45   memcpy(header, XZ_SIG, XZ_SIG_SIZE);
     46   header[XZ_SIG_SIZE] = (Byte)(f >> 8);
     47   header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
     48   crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
     49   SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
     50   return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
     51 }
     52 
     53 SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
     54 {
     55   Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
     56 
     57   unsigned pos = 1;
     58   int numFilters, i;
     59   header[pos++] = p->flags;
     60 
     61   if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
     62   if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
     63   numFilters = XzBlock_GetNumFilters(p);
     64   for (i = 0; i < numFilters; i++)
     65   {
     66     const CXzFilter *f = &p->filters[i];
     67     pos += Xz_WriteVarInt(header + pos, f->id);
     68     pos += Xz_WriteVarInt(header + pos, f->propsSize);
     69     memcpy(header + pos, f->props, f->propsSize);
     70     pos += f->propsSize;
     71   }
     72   while((pos & 3) != 0)
     73     header[pos++] = 0;
     74   header[0] = (Byte)(pos >> 2);
     75   SetUi32(header + pos, CrcCalc(header, pos));
     76   return WriteBytes(s, header, pos + 4);
     77 }
     78 
     79 SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
     80 {
     81   Byte buf[32];
     82   UInt64 globalPos;
     83   {
     84     UInt32 crc = CRC_INIT_VAL;
     85     unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
     86     size_t i;
     87 
     88     globalPos = pos;
     89     buf[0] = 0;
     90     RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
     91     for (i = 0; i < p->numBlocks; i++)
     92     {
     93       const CXzBlockSizes *block = &p->blocks[i];
     94       pos = Xz_WriteVarInt(buf, block->totalSize);
     95       pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
     96       globalPos += pos;
     97       RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
     98     }
     99     pos = ((unsigned)globalPos & 3);
    100     if (pos != 0)
    101     {
    102       buf[0] = buf[1] = buf[2] = 0;
    103       RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
    104       globalPos += 4 - pos;
    105     }
    106     {
    107       SetUi32(buf, CRC_GET_DIGEST(crc));
    108       RINOK(WriteBytes(s, buf, 4));
    109       globalPos += 4;
    110     }
    111   }
    112 
    113   {
    114     UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
    115     SetUi32(buf + 4, indexSize);
    116     buf[8] = (Byte)(p->flags >> 8);
    117     buf[9] = (Byte)(p->flags & 0xFF);
    118     SetUi32(buf, CrcCalc(buf + 4, 6));
    119     memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
    120     return WriteBytes(s, buf, 12);
    121   }
    122 }
    123 
    124 SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
    125 {
    126   if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks)
    127   {
    128     size_t num = (p->numBlocks + 1) * 2;
    129     size_t newSize = sizeof(CXzBlockSizes) * num;
    130     CXzBlockSizes *blocks;
    131     if (newSize / sizeof(CXzBlockSizes) != num)
    132       return SZ_ERROR_MEM;
    133     blocks = alloc->Alloc(alloc, newSize);
    134     if (blocks == 0)
    135       return SZ_ERROR_MEM;
    136     if (p->numBlocks != 0)
    137     {
    138       memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
    139       Xz_Free(p, alloc);
    140     }
    141     p->blocks = blocks;
    142     p->numBlocksAllocated = num;
    143   }
    144   {
    145     CXzBlockSizes *block = &p->blocks[p->numBlocks++];
    146     block->totalSize = totalSize;
    147     block->unpackSize = unpackSize;
    148   }
    149   return SZ_OK;
    150 }
    151 
    152 /* ---------- CSeqCheckInStream ---------- */
    153 
    154 typedef struct
    155 {
    156   ISeqInStream p;
    157   ISeqInStream *realStream;
    158   UInt64 processed;
    159   CXzCheck check;
    160 } CSeqCheckInStream;
    161 
    162 void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode)
    163 {
    164   p->processed = 0;
    165   XzCheck_Init(&p->check, mode);
    166 }
    167 
    168 void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
    169 {
    170   XzCheck_Final(&p->check, digest);
    171 }
    172 
    173 static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
    174 {
    175   CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
    176   SRes res = p->realStream->Read(p->realStream, data, size);
    177   XzCheck_Update(&p->check, data, *size);
    178   p->processed += *size;
    179   return res;
    180 }
    181 
    182 /* ---------- CSeqSizeOutStream ---------- */
    183 
    184 typedef struct
    185 {
    186   ISeqOutStream p;
    187   ISeqOutStream *realStream;
    188   UInt64 processed;
    189 } CSeqSizeOutStream;
    190 
    191 static size_t MyWrite(void *pp, const void *data, size_t size)
    192 {
    193   CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
    194   size = p->realStream->Write(p->realStream, data, size);
    195   p->processed += size;
    196   return size;
    197 }
    198 
    199 /* ---------- CSeqInFilter ---------- */
    200 
    201 /*
    202 typedef struct _IFilter
    203 {
    204   void *p;
    205   void (*Free)(void *p, ISzAlloc *alloc);
    206   SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAlloc *alloc);
    207   void (*Init)(void *p);
    208   size_t (*Filter)(void *p, Byte *data, SizeT destLen);
    209 } IFilter;
    210 
    211 #define FILT_BUF_SIZE (1 << 19)
    212 
    213 typedef struct
    214 {
    215   ISeqInStream p;
    216   ISeqInStream *realStream;
    217   UInt32 x86State;
    218   UInt32 ip;
    219   UInt64 processed;
    220   CXzCheck check;
    221   Byte buf[FILT_BUF_SIZE];
    222   UInt32 bufferPos;
    223   UInt32 convertedPosBegin;
    224   UInt32 convertedPosEnd;
    225   IFilter *filter;
    226 } CSeqInFilter;
    227 
    228 static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
    229 {
    230   CSeqInFilter *p = (CSeqInFilter *)pp;
    231   size_t remSize = *size;
    232   *size = 0;
    233 
    234   while (remSize > 0)
    235   {
    236     int i;
    237     if (p->convertedPosBegin != p->convertedPosEnd)
    238     {
    239       UInt32 sizeTemp = p->convertedPosEnd - p->convertedPosBegin;
    240       if (remSize < sizeTemp)
    241         sizeTemp = (UInt32)remSize;
    242       memmove(data, p->buf + p->convertedPosBegin, sizeTemp);
    243       p->convertedPosBegin += sizeTemp;
    244       data = (void *)((Byte *)data + sizeTemp);
    245       remSize -= sizeTemp;
    246       *size += sizeTemp;
    247       break;
    248     }
    249     for (i = 0; p->convertedPosEnd + i < p->bufferPos; i++)
    250       p->buf[i] = p->buf[i + p->convertedPosEnd];
    251     p->bufferPos = i;
    252     p->convertedPosBegin = p->convertedPosEnd = 0;
    253     {
    254       size_t processedSizeTemp = FILT_BUF_SIZE - p->bufferPos;
    255       RINOK(p->realStream->Read(p->realStream, p->buf + p->bufferPos, &processedSizeTemp));
    256       p->bufferPos = p->bufferPos + (UInt32)processedSizeTemp;
    257     }
    258     p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos);
    259     if (p->convertedPosEnd == 0)
    260     {
    261       if (p->bufferPos == 0)
    262         break;
    263       else
    264       {
    265         p->convertedPosEnd = p->bufferPos;
    266         continue;
    267       }
    268     }
    269     if (p->convertedPosEnd > p->bufferPos)
    270     {
    271       for (; p->bufferPos < p->convertedPosEnd; p->bufferPos++)
    272         p->buf[p->bufferPos] = 0;
    273       p->convertedPosEnd = (UInt32)p->filter->Filter(p->filter->p, p->buf, p->bufferPos);
    274     }
    275   }
    276   return SZ_OK;
    277 }
    278 */
    279 
    280 /*
    281 typedef struct
    282 {
    283   ISeqInStream p;
    284   ISeqInStream *realStream;
    285   CMixCoder mixCoder;
    286   Byte buf[FILT_BUF_SIZE];
    287   UInt32 bufPos;
    288   UInt32 bufSize;
    289 } CMixCoderSeqInStream;
    290 
    291 static SRes CMixCoderSeqInStream_Read(void *pp, void *data, size_t *size)
    292 {
    293   CMixCoderSeqInStream *p = (CMixCoderSeqInStream *)pp;
    294   SRes res = SZ_OK;
    295   size_t remSize = *size;
    296   *size = 0;
    297   while (remSize > 0)
    298   {
    299     if (p->bufPos == p->bufSize)
    300     {
    301       size_t curSize;
    302       p->bufPos = p->bufSize = 0;
    303       if (*size != 0)
    304         break;
    305       curSize = FILT_BUF_SIZE;
    306       RINOK(p->realStream->Read(p->realStream, p->buf, &curSize));
    307       p->bufSize = (UInt32)curSize;
    308     }
    309     {
    310       SizeT destLen = remSize;
    311       SizeT srcLen = p->bufSize - p->bufPos;
    312       res = MixCoder_Code(&p->mixCoder, data, &destLen, p->buf + p->bufPos, &srcLen, 0);
    313       data = (void *)((Byte *)data + destLen);
    314       remSize -= destLen;
    315       *size += destLen;
    316       p->bufPos += srcLen;
    317     }
    318   }
    319   return res;
    320 }
    321 */
    322 
    323 #ifdef USE_SUBBLOCK
    324 typedef struct
    325 {
    326   ISeqInStream p;
    327   CSubblockEnc sb;
    328   UInt64 processed;
    329 } CSbEncInStream;
    330 
    331 void SbEncInStream_Init(CSbEncInStream *p)
    332 {
    333   p->processed = 0;
    334   SubblockEnc_Init(&p->sb);
    335 }
    336 
    337 static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
    338 {
    339   CSbEncInStream *p = (CSbEncInStream *)pp;
    340   SRes res = SubblockEnc_Read(&p->sb, data, size);
    341   p->processed += *size;
    342   return res;
    343 }
    344 #endif
    345 
    346 typedef struct
    347 {
    348   /* CMixCoderSeqInStream inStream; */
    349   CLzma2EncHandle lzma2;
    350   #ifdef USE_SUBBLOCK
    351   CSbEncInStream sb;
    352   #endif
    353   ISzAlloc *alloc;
    354   ISzAlloc *bigAlloc;
    355 } CLzma2WithFilters;
    356 
    357 
    358 static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
    359 {
    360   p->alloc = alloc;
    361   p->bigAlloc = bigAlloc;
    362   p->lzma2 = NULL;
    363   #ifdef USE_SUBBLOCK
    364   p->sb.p.Read = SbEncInStream_Read;
    365   SubblockEnc_Construct(&p->sb.sb, p->alloc);
    366   #endif
    367 }
    368 
    369 static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
    370 {
    371   p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
    372   if (p->lzma2 == 0)
    373     return SZ_ERROR_MEM;
    374   return SZ_OK;
    375 }
    376 
    377 static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
    378 {
    379   #ifdef USE_SUBBLOCK
    380   SubblockEnc_Free(&p->sb.sb);
    381   #endif
    382   if (p->lzma2)
    383   {
    384     Lzma2Enc_Destroy(p->lzma2);
    385     p->lzma2 = NULL;
    386   }
    387 }
    388 
    389 static SRes Xz_Compress(CXzStream *xz,
    390     CLzma2WithFilters *lzmaf,
    391     ISeqOutStream *outStream,
    392     ISeqInStream *inStream,
    393     const CLzma2EncProps *lzma2Props,
    394     Bool useSubblock,
    395     ICompressProgress *progress)
    396 {
    397   xz->flags = XZ_CHECK_CRC32;
    398 
    399   RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, lzma2Props));
    400   RINOK(Xz_WriteHeader(xz->flags, outStream));
    401 
    402   {
    403     CSeqCheckInStream checkInStream;
    404     CSeqSizeOutStream seqSizeOutStream;
    405     CXzBlock block;
    406     int filterIndex = 0;
    407 
    408     XzBlock_ClearFlags(&block);
    409     XzBlock_SetNumFilters(&block, 1 + (useSubblock ? 1 : 0));
    410 
    411     if (useSubblock)
    412     {
    413       CXzFilter *f = &block.filters[filterIndex++];
    414       f->id = XZ_ID_Subblock;
    415       f->propsSize = 0;
    416     }
    417 
    418     {
    419       CXzFilter *f = &block.filters[filterIndex++];
    420       f->id = XZ_ID_LZMA2;
    421       f->propsSize = 1;
    422       f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
    423     }
    424 
    425     seqSizeOutStream.p.Write = MyWrite;
    426     seqSizeOutStream.realStream = outStream;
    427     seqSizeOutStream.processed = 0;
    428 
    429     RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
    430 
    431     checkInStream.p.Read = SeqCheckInStream_Read;
    432     checkInStream.realStream = inStream;
    433     SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
    434 
    435     #ifdef USE_SUBBLOCK
    436     if (useSubblock)
    437     {
    438       lzmaf->sb.sb.inStream = &checkInStream.p;
    439       SubblockEnc_Init(&lzmaf->sb.sb);
    440     }
    441     #endif
    442 
    443     {
    444       UInt64 packPos = seqSizeOutStream.processed;
    445       SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
    446         #ifdef USE_SUBBLOCK
    447         useSubblock ? &lzmaf->sb.p:
    448         #endif
    449         &checkInStream.p,
    450         progress);
    451       RINOK(res);
    452       block.unpackSize = checkInStream.processed;
    453       block.packSize = seqSizeOutStream.processed - packPos;
    454     }
    455 
    456     {
    457       unsigned padSize = 0;
    458       Byte buf[128];
    459       while((((unsigned)block.packSize + padSize) & 3) != 0)
    460         buf[padSize++] = 0;
    461       SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
    462       RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
    463       RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
    464     }
    465   }
    466   return Xz_WriteFooter(xz, outStream);
    467 }
    468 
    469 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
    470     const CLzma2EncProps *lzma2Props, Bool useSubblock,
    471     ICompressProgress *progress)
    472 {
    473   SRes res;
    474   CXzStream xz;
    475   CLzma2WithFilters lzmaf;
    476   Xz_Construct(&xz);
    477   Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
    478   res = Lzma2WithFilters_Create(&lzmaf);
    479   if (res == SZ_OK)
    480     res = Xz_Compress(&xz, &lzmaf, outStream, inStream,
    481         lzma2Props, useSubblock, progress);
    482   Lzma2WithFilters_Free(&lzmaf);
    483   Xz_Free(&xz, &g_Alloc);
    484   return res;
    485 }
    486 
    487 SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
    488 {
    489   SRes res;
    490   CXzStream xz;
    491   Xz_Construct(&xz);
    492   res = Xz_WriteHeader(xz.flags, outStream);
    493   if (res == SZ_OK)
    494     res = Xz_WriteFooter(&xz, outStream);
    495   Xz_Free(&xz, &g_Alloc);
    496   return res;
    497 }
    498