Home | History | Annotate | Download | only in C 
      1 /* XzEnc.c -- Xz Encode
      2 2014-12-30 : Igor Pavlov : Public domain */
      3 
      4 #include "Precomp.h"
      5 
      6 #include <stdlib.h>
      7 #include <string.h>
      8 
      9 #include "7zCrc.h"
     10 #include "Alloc.h"
     11 #include "Bra.h"
     12 #include "CpuArch.h"
     13 #ifdef USE_SUBBLOCK
     14 #include "Bcj3Enc.c"
     15 #include "SbFind.c"
     16 #include "SbEnc.c"
     17 #endif
     18 
     19 #include "XzEnc.h"
     20 
     21 static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); }
     22 static void SzBigFree(void *p, void *address) { p = p; BigFree(address); }
     23 static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
     24 
     25 static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
     26 static void SzFree(void *p, void *address) { p = p; MyFree(address); }
     27 static ISzAlloc g_Alloc = { SzAlloc, SzFree };
     28 
     29 #define XzBlock_ClearFlags(p)       (p)->flags = 0;
     30 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
     31 #define XzBlock_SetHasPackSize(p)   (p)->flags |= XZ_BF_PACK_SIZE;
     32 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
     33 
     34 static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
     35 {
     36   return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
     37 }
     38 
     39 static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
     40 {
     41   *crc = CrcUpdate(*crc, buf, size);
     42   return WriteBytes(s, buf, size);
     43 }
     44 
     45 SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
     46 {
     47   UInt32 crc;
     48   Byte header[XZ_STREAM_HEADER_SIZE];
     49   memcpy(header, XZ_SIG, XZ_SIG_SIZE);
     50   header[XZ_SIG_SIZE] = (Byte)(f >> 8);
     51   header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
     52   crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
     53   SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
     54   return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
     55 }
     56 
     57 SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
     58 {
     59   Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
     60 
     61   unsigned pos = 1;
     62   int numFilters, i;
     63   header[pos++] = p->flags;
     64 
     65   if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
     66   if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
     67   numFilters = XzBlock_GetNumFilters(p);
     68   for (i = 0; i < numFilters; i++)
     69   {
     70     const CXzFilter *f = &p->filters[i];
     71     pos += Xz_WriteVarInt(header + pos, f->id);
     72     pos += Xz_WriteVarInt(header + pos, f->propsSize);
     73     memcpy(header + pos, f->props, f->propsSize);
     74     pos += f->propsSize;
     75   }
     76   while((pos & 3) != 0)
     77     header[pos++] = 0;
     78   header[0] = (Byte)(pos >> 2);
     79   SetUi32(header + pos, CrcCalc(header, pos));
     80   return WriteBytes(s, header, pos + 4);
     81 }
     82 
     83 SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
     84 {
     85   Byte buf[32];
     86   UInt64 globalPos;
     87   {
     88     UInt32 crc = CRC_INIT_VAL;
     89     unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
     90     size_t i;
     91 
     92     globalPos = pos;
     93     buf[0] = 0;
     94     RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
     95     for (i = 0; i < p->numBlocks; i++)
     96     {
     97       const CXzBlockSizes *block = &p->blocks[i];
     98       pos = Xz_WriteVarInt(buf, block->totalSize);
     99       pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
    100       globalPos += pos;
    101       RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
    102     }
    103     pos = ((unsigned)globalPos & 3);
    104     if (pos != 0)
    105     {
    106       buf[0] = buf[1] = buf[2] = 0;
    107       RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
    108       globalPos += 4 - pos;
    109     }
    110     {
    111       SetUi32(buf, CRC_GET_DIGEST(crc));
    112       RINOK(WriteBytes(s, buf, 4));
    113       globalPos += 4;
    114     }
    115   }
    116 
    117   {
    118     UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
    119     SetUi32(buf + 4, indexSize);
    120     buf[8] = (Byte)(p->flags >> 8);
    121     buf[9] = (Byte)(p->flags & 0xFF);
    122     SetUi32(buf, CrcCalc(buf + 4, 6));
    123     memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
    124     return WriteBytes(s, buf, 12);
    125   }
    126 }
    127 
    128 SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
    129 {
    130   if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks)
    131   {
    132     size_t num = (p->numBlocks + 1) * 2;
    133     size_t newSize = sizeof(CXzBlockSizes) * num;
    134     CXzBlockSizes *blocks;
    135     if (newSize / sizeof(CXzBlockSizes) != num)
    136       return SZ_ERROR_MEM;
    137     blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize);
    138     if (blocks == 0)
    139       return SZ_ERROR_MEM;
    140     if (p->numBlocks != 0)
    141     {
    142       memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
    143       Xz_Free(p, alloc);
    144     }
    145     p->blocks = blocks;
    146     p->numBlocksAllocated = num;
    147   }
    148   {
    149     CXzBlockSizes *block = &p->blocks[p->numBlocks++];
    150     block->totalSize = totalSize;
    151     block->unpackSize = unpackSize;
    152   }
    153   return SZ_OK;
    154 }
    155 
    156 /* ---------- CSeqCheckInStream ---------- */
    157 
    158 typedef struct
    159 {
    160   ISeqInStream p;
    161   ISeqInStream *realStream;
    162   UInt64 processed;
    163   CXzCheck check;
    164 } CSeqCheckInStream;
    165 
    166 void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode)
    167 {
    168   p->processed = 0;
    169   XzCheck_Init(&p->check, mode);
    170 }
    171 
    172 void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
    173 {
    174   XzCheck_Final(&p->check, digest);
    175 }
    176 
    177 static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
    178 {
    179   CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
    180   SRes res = p->realStream->Read(p->realStream, data, size);
    181   XzCheck_Update(&p->check, data, *size);
    182   p->processed += *size;
    183   return res;
    184 }
    185 
    186 /* ---------- CSeqSizeOutStream ---------- */
    187 
    188 typedef struct
    189 {
    190   ISeqOutStream p;
    191   ISeqOutStream *realStream;
    192   UInt64 processed;
    193 } CSeqSizeOutStream;
    194 
    195 static size_t MyWrite(void *pp, const void *data, size_t size)
    196 {
    197   CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
    198   size = p->realStream->Write(p->realStream, data, size);
    199   p->processed += size;
    200   return size;
    201 }
    202 
    203 /* ---------- CSeqInFilter ---------- */
    204 
    205 #define FILTER_BUF_SIZE (1 << 20)
    206 
    207 typedef struct
    208 {
    209   ISeqInStream p;
    210   ISeqInStream *realStream;
    211   IStateCoder StateCoder;
    212   Byte *buf;
    213   size_t curPos;
    214   size_t endPos;
    215   int srcWasFinished;
    216 } CSeqInFilter;
    217 
    218 static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
    219 {
    220   CSeqInFilter *p = (CSeqInFilter *)pp;
    221   size_t sizeOriginal = *size;
    222   if (sizeOriginal == 0)
    223     return SZ_OK;
    224   *size = 0;
    225   for (;;)
    226   {
    227     if (!p->srcWasFinished && p->curPos == p->endPos)
    228     {
    229       p->curPos = 0;
    230       p->endPos = FILTER_BUF_SIZE;
    231       RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos));
    232       if (p->endPos == 0)
    233         p->srcWasFinished = 1;
    234     }
    235     {
    236       SizeT srcLen = p->endPos - p->curPos;
    237       int wasFinished;
    238       SRes res;
    239       *size = sizeOriginal;
    240       res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen,
    241         p->srcWasFinished, CODER_FINISH_ANY, &wasFinished);
    242       p->curPos += srcLen;
    243       if (*size != 0 || srcLen == 0 || res != 0)
    244         return res;
    245     }
    246   }
    247 }
    248 
    249 static void SeqInFilter_Construct(CSeqInFilter *p)
    250 {
    251   p->buf = NULL;
    252   p->p.Read = SeqInFilter_Read;
    253 }
    254 
    255 static void SeqInFilter_Free(CSeqInFilter *p)
    256 {
    257   if (p->buf)
    258   {
    259     g_Alloc.Free(&g_Alloc, p->buf);
    260     p->buf = NULL;
    261   }
    262 }
    263 
    264 SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc);
    265 
    266 static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props)
    267 {
    268   if (!p->buf)
    269   {
    270     p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE);
    271     if (!p->buf)
    272       return SZ_ERROR_MEM;
    273   }
    274   p->curPos = p->endPos = 0;
    275   p->srcWasFinished = 0;
    276   RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc));
    277   RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc));
    278   p->StateCoder.Init(p->StateCoder.p);
    279   return SZ_OK;
    280 }
    281 
    282 /* ---------- CSbEncInStream ---------- */
    283 
    284 #ifdef USE_SUBBLOCK
    285 
    286 typedef struct
    287 {
    288   ISeqInStream p;
    289   ISeqInStream *inStream;
    290   CSbEnc enc;
    291 } CSbEncInStream;
    292 
    293 static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
    294 {
    295   CSbEncInStream *p = (CSbEncInStream *)pp;
    296   size_t sizeOriginal = *size;
    297   if (sizeOriginal == 0)
    298     return S_OK;
    299   for (;;)
    300   {
    301     if (p->enc.needRead && !p->enc.readWasFinished)
    302     {
    303       size_t processed = p->enc.needReadSizeMax;
    304       RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed));
    305       p->enc.readPos += processed;
    306       if (processed == 0)
    307       {
    308         p->enc.readWasFinished = True;
    309         p->enc.isFinalFinished = True;
    310       }
    311       p->enc.needRead = False;
    312     }
    313     *size = sizeOriginal;
    314     RINOK(SbEnc_Read(&p->enc, data, size));
    315     if (*size != 0 || !p->enc.needRead)
    316       return S_OK;
    317   }
    318 }
    319 
    320 void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc)
    321 {
    322   SbEnc_Construct(&p->enc, alloc);
    323   p->p.Read = SbEncInStream_Read;
    324 }
    325 
    326 SRes SbEncInStream_Init(CSbEncInStream *p)
    327 {
    328   return SbEnc_Init(&p->enc);
    329 }
    330 
    331 void SbEncInStream_Free(CSbEncInStream *p)
    332 {
    333   SbEnc_Free(&p->enc);
    334 }
    335 
    336 #endif
    337 
    338 
    339 typedef struct
    340 {
    341   CLzma2EncHandle lzma2;
    342   #ifdef USE_SUBBLOCK
    343   CSbEncInStream sb;
    344   #endif
    345   CSeqInFilter filter;
    346   ISzAlloc *alloc;
    347   ISzAlloc *bigAlloc;
    348 } CLzma2WithFilters;
    349 
    350 
    351 static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
    352 {
    353   p->alloc = alloc;
    354   p->bigAlloc = bigAlloc;
    355   p->lzma2 = NULL;
    356   #ifdef USE_SUBBLOCK
    357   SbEncInStream_Construct(&p->sb, alloc);
    358   #endif
    359   SeqInFilter_Construct(&p->filter);
    360 }
    361 
    362 static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
    363 {
    364   p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
    365   if (p->lzma2 == 0)
    366     return SZ_ERROR_MEM;
    367   return SZ_OK;
    368 }
    369 
    370 static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
    371 {
    372   SeqInFilter_Free(&p->filter);
    373   #ifdef USE_SUBBLOCK
    374   SbEncInStream_Free(&p->sb);
    375   #endif
    376   if (p->lzma2)
    377   {
    378     Lzma2Enc_Destroy(p->lzma2);
    379     p->lzma2 = NULL;
    380   }
    381 }
    382 
    383 void XzProps_Init(CXzProps *p)
    384 {
    385   p->lzma2Props = 0;
    386   p->filterProps = 0;
    387   p->checkId = XZ_CHECK_CRC32;
    388 }
    389 
    390 void XzFilterProps_Init(CXzFilterProps *p)
    391 {
    392   p->id = 0;
    393   p->delta = 0;
    394   p->ip= 0;
    395   p->ipDefined = False;
    396 }
    397 
    398 static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf,
    399     ISeqOutStream *outStream, ISeqInStream *inStream,
    400     const CXzProps *props, ICompressProgress *progress)
    401 {
    402   xz->flags = (Byte)props->checkId;
    403 
    404   RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props));
    405   RINOK(Xz_WriteHeader(xz->flags, outStream));
    406 
    407   {
    408     CSeqCheckInStream checkInStream;
    409     CSeqSizeOutStream seqSizeOutStream;
    410     CXzBlock block;
    411     int filterIndex = 0;
    412     CXzFilter *filter = NULL;
    413     const CXzFilterProps *fp = props->filterProps;
    414 
    415     XzBlock_ClearFlags(&block);
    416     XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0));
    417 
    418     if (fp)
    419     {
    420       filter = &block.filters[filterIndex++];
    421       filter->id = fp->id;
    422       filter->propsSize = 0;
    423       if (fp->id == XZ_ID_Delta)
    424       {
    425         filter->props[0] = (Byte)(fp->delta - 1);
    426         filter->propsSize = 1;
    427       }
    428       else if (fp->ipDefined)
    429       {
    430         SetUi32(filter->props, fp->ip);
    431         filter->propsSize = 4;
    432       }
    433     }
    434 
    435     {
    436       CXzFilter *f = &block.filters[filterIndex++];
    437       f->id = XZ_ID_LZMA2;
    438       f->propsSize = 1;
    439       f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
    440     }
    441 
    442     seqSizeOutStream.p.Write = MyWrite;
    443     seqSizeOutStream.realStream = outStream;
    444     seqSizeOutStream.processed = 0;
    445 
    446     RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
    447 
    448     checkInStream.p.Read = SeqCheckInStream_Read;
    449     checkInStream.realStream = inStream;
    450     SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
    451 
    452     if (fp)
    453     {
    454       #ifdef USE_SUBBLOCK
    455       if (fp->id == XZ_ID_Subblock)
    456       {
    457         lzmaf->sb.inStream = &checkInStream.p;
    458         RINOK(SbEncInStream_Init(&lzmaf->sb));
    459       }
    460       else
    461       #endif
    462       {
    463         lzmaf->filter.realStream = &checkInStream.p;
    464         RINOK(SeqInFilter_Init(&lzmaf->filter, filter));
    465       }
    466     }
    467 
    468     {
    469       UInt64 packPos = seqSizeOutStream.processed;
    470       SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
    471         fp ?
    472         #ifdef USE_SUBBLOCK
    473         (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p:
    474         #endif
    475         &lzmaf->filter.p:
    476         &checkInStream.p,
    477         progress);
    478       RINOK(res);
    479       block.unpackSize = checkInStream.processed;
    480       block.packSize = seqSizeOutStream.processed - packPos;
    481     }
    482 
    483     {
    484       unsigned padSize = 0;
    485       Byte buf[128];
    486       while((((unsigned)block.packSize + padSize) & 3) != 0)
    487         buf[padSize++] = 0;
    488       SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
    489       RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
    490       RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
    491     }
    492   }
    493   return Xz_WriteFooter(xz, outStream);
    494 }
    495 
    496 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
    497     const CXzProps *props, ICompressProgress *progress)
    498 {
    499   SRes res;
    500   CXzStream xz;
    501   CLzma2WithFilters lzmaf;
    502   Xz_Construct(&xz);
    503   Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
    504   res = Lzma2WithFilters_Create(&lzmaf);
    505   if (res == SZ_OK)
    506     res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress);
    507   Lzma2WithFilters_Free(&lzmaf);
    508   Xz_Free(&xz, &g_Alloc);
    509   return res;
    510 }
    511 
    512 SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
    513 {
    514   SRes res;
    515   CXzStream xz;
    516   Xz_Construct(&xz);
    517   res = Xz_WriteHeader(xz.flags, outStream);
    518   if (res == SZ_OK)
    519     res = Xz_WriteFooter(&xz, outStream);
    520   Xz_Free(&xz, &g_Alloc);
    521   return res;
    522 }
    523