Home | History | Annotate | Download | only in C 
      1 /* Xz.h - Xz interface
      2 2018-07-04 : Igor Pavlov : Public domain */
      3 
      4 #ifndef __XZ_H
      5 #define __XZ_H
      6 
      7 #include "Sha256.h"
      8 
      9 EXTERN_C_BEGIN
     10 
     11 #define XZ_ID_Subblock 1
     12 #define XZ_ID_Delta 3
     13 #define XZ_ID_X86 4
     14 #define XZ_ID_PPC 5
     15 #define XZ_ID_IA64 6
     16 #define XZ_ID_ARM 7
     17 #define XZ_ID_ARMT 8
     18 #define XZ_ID_SPARC 9
     19 #define XZ_ID_LZMA2 0x21
     20 
     21 unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value);
     22 unsigned Xz_WriteVarInt(Byte *buf, UInt64 v);
     23 
     24 /* ---------- xz block ---------- */
     25 
     26 #define XZ_BLOCK_HEADER_SIZE_MAX 1024
     27 
     28 #define XZ_NUM_FILTERS_MAX 4
     29 #define XZ_BF_NUM_FILTERS_MASK 3
     30 #define XZ_BF_PACK_SIZE (1 << 6)
     31 #define XZ_BF_UNPACK_SIZE (1 << 7)
     32 
     33 #define XZ_FILTER_PROPS_SIZE_MAX 20
     34 
     35 typedef struct
     36 {
     37   UInt64 id;
     38   UInt32 propsSize;
     39   Byte props[XZ_FILTER_PROPS_SIZE_MAX];
     40 } CXzFilter;
     41 
     42 typedef struct
     43 {
     44   UInt64 packSize;
     45   UInt64 unpackSize;
     46   Byte flags;
     47   CXzFilter filters[XZ_NUM_FILTERS_MAX];
     48 } CXzBlock;
     49 
     50 #define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1)
     51 #define XzBlock_HasPackSize(p)   (((p)->flags & XZ_BF_PACK_SIZE) != 0)
     52 #define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0)
     53 #define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0)
     54 
     55 SRes XzBlock_Parse(CXzBlock *p, const Byte *header);
     56 SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes);
     57 
     58 /* ---------- xz stream ---------- */
     59 
     60 #define XZ_SIG_SIZE 6
     61 #define XZ_FOOTER_SIG_SIZE 2
     62 
     63 extern const Byte XZ_SIG[XZ_SIG_SIZE];
     64 
     65 /*
     66 extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE];
     67 */
     68 
     69 #define XZ_FOOTER_SIG_0 'Y'
     70 #define XZ_FOOTER_SIG_1 'Z'
     71 
     72 #define XZ_STREAM_FLAGS_SIZE 2
     73 #define XZ_STREAM_CRC_SIZE 4
     74 
     75 #define XZ_STREAM_HEADER_SIZE (XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE + XZ_STREAM_CRC_SIZE)
     76 #define XZ_STREAM_FOOTER_SIZE (XZ_FOOTER_SIG_SIZE + XZ_STREAM_FLAGS_SIZE + XZ_STREAM_CRC_SIZE + 4)
     77 
     78 #define XZ_CHECK_MASK 0xF
     79 #define XZ_CHECK_NO 0
     80 #define XZ_CHECK_CRC32 1
     81 #define XZ_CHECK_CRC64 4
     82 #define XZ_CHECK_SHA256 10
     83 
     84 typedef struct
     85 {
     86   unsigned mode;
     87   UInt32 crc;
     88   UInt64 crc64;
     89   CSha256 sha;
     90 } CXzCheck;
     91 
     92 void XzCheck_Init(CXzCheck *p, unsigned mode);
     93 void XzCheck_Update(CXzCheck *p, const void *data, size_t size);
     94 int XzCheck_Final(CXzCheck *p, Byte *digest);
     95 
     96 typedef UInt16 CXzStreamFlags;
     97 
     98 #define XzFlags_IsSupported(f) ((f) <= XZ_CHECK_MASK)
     99 #define XzFlags_GetCheckType(f) ((f) & XZ_CHECK_MASK)
    100 #define XzFlags_HasDataCrc32(f) (Xz_GetCheckType(f) == XZ_CHECK_CRC32)
    101 unsigned XzFlags_GetCheckSize(CXzStreamFlags f);
    102 
    103 SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf);
    104 SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream);
    105 
    106 typedef struct
    107 {
    108   UInt64 unpackSize;
    109   UInt64 totalSize;
    110 } CXzBlockSizes;
    111 
    112 typedef struct
    113 {
    114   CXzStreamFlags flags;
    115   size_t numBlocks;
    116   CXzBlockSizes *blocks;
    117   UInt64 startOffset;
    118 } CXzStream;
    119 
    120 void Xz_Construct(CXzStream *p);
    121 void Xz_Free(CXzStream *p, ISzAllocPtr alloc);
    122 
    123 #define XZ_SIZE_OVERFLOW ((UInt64)(Int64)-1)
    124 
    125 UInt64 Xz_GetUnpackSize(const CXzStream *p);
    126 UInt64 Xz_GetPackSize(const CXzStream *p);
    127 
    128 typedef struct
    129 {
    130   size_t num;
    131   size_t numAllocated;
    132   CXzStream *streams;
    133 } CXzs;
    134 
    135 void Xzs_Construct(CXzs *p);
    136 void Xzs_Free(CXzs *p, ISzAllocPtr alloc);
    137 SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc);
    138 
    139 UInt64 Xzs_GetNumBlocks(const CXzs *p);
    140 UInt64 Xzs_GetUnpackSize(const CXzs *p);
    141 
    142 
    143 // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder
    144 
    145 typedef enum
    146 {
    147   CODER_STATUS_NOT_SPECIFIED,               /* use main error code instead */
    148   CODER_STATUS_FINISHED_WITH_MARK,          /* stream was finished with end mark. */
    149   CODER_STATUS_NOT_FINISHED,                /* stream was not finished */
    150   CODER_STATUS_NEEDS_MORE_INPUT             /* you must provide more input bytes */
    151 } ECoderStatus;
    152 
    153 
    154 // ECoderFinishMode values are identical to ELzmaFinishMode
    155 
    156 typedef enum
    157 {
    158   CODER_FINISH_ANY,   /* finish at any point */
    159   CODER_FINISH_END    /* block must be finished at the end */
    160 } ECoderFinishMode;
    161 
    162 
    163 typedef struct _IStateCoder
    164 {
    165   void *p;
    166   void (*Free)(void *p, ISzAllocPtr alloc);
    167   SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc);
    168   void (*Init)(void *p);
    169   SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
    170       int srcWasFinished, ECoderFinishMode finishMode,
    171       // int *wasFinished,
    172       ECoderStatus *status);
    173   SizeT (*Filter)(void *p, Byte *data, SizeT size);
    174 } IStateCoder;
    175 
    176 
    177 
    178 #define MIXCODER_NUM_FILTERS_MAX 4
    179 
    180 typedef struct
    181 {
    182   ISzAllocPtr alloc;
    183   Byte *buf;
    184   unsigned numCoders;
    185 
    186   Byte *outBuf;
    187   size_t outBufSize;
    188   size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode)
    189   BoolInt wasFinished;
    190   SRes res;
    191   ECoderStatus status;
    192   // BoolInt SingleBufMode;
    193 
    194   int finished[MIXCODER_NUM_FILTERS_MAX - 1];
    195   size_t pos[MIXCODER_NUM_FILTERS_MAX - 1];
    196   size_t size[MIXCODER_NUM_FILTERS_MAX - 1];
    197   UInt64 ids[MIXCODER_NUM_FILTERS_MAX];
    198   SRes results[MIXCODER_NUM_FILTERS_MAX];
    199   IStateCoder coders[MIXCODER_NUM_FILTERS_MAX];
    200 } CMixCoder;
    201 
    202 
    203 typedef enum
    204 {
    205   XZ_STATE_STREAM_HEADER,
    206   XZ_STATE_STREAM_INDEX,
    207   XZ_STATE_STREAM_INDEX_CRC,
    208   XZ_STATE_STREAM_FOOTER,
    209   XZ_STATE_STREAM_PADDING,
    210   XZ_STATE_BLOCK_HEADER,
    211   XZ_STATE_BLOCK,
    212   XZ_STATE_BLOCK_FOOTER
    213 } EXzState;
    214 
    215 
    216 typedef struct
    217 {
    218   EXzState state;
    219   UInt32 pos;
    220   unsigned alignPos;
    221   unsigned indexPreSize;
    222 
    223   CXzStreamFlags streamFlags;
    224 
    225   UInt32 blockHeaderSize;
    226   UInt64 packSize;
    227   UInt64 unpackSize;
    228 
    229   UInt64 numBlocks; // number of finished blocks in current stream
    230   UInt64 indexSize;
    231   UInt64 indexPos;
    232   UInt64 padSize;
    233 
    234   UInt64 numStartedStreams;
    235   UInt64 numFinishedStreams;
    236   UInt64 numTotalBlocks;
    237 
    238   UInt32 crc;
    239   CMixCoder decoder;
    240   CXzBlock block;
    241   CXzCheck check;
    242   CSha256 sha;
    243 
    244   BoolInt parseMode;
    245   BoolInt headerParsedOk;
    246   BoolInt decodeToStreamSignature;
    247   unsigned decodeOnlyOneBlock;
    248 
    249   Byte *outBuf;
    250   size_t outBufSize;
    251   size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked
    252 
    253   Byte shaDigest[SHA256_DIGEST_SIZE];
    254   Byte buf[XZ_BLOCK_HEADER_SIZE_MAX];
    255 } CXzUnpacker;
    256 
    257 /* alloc : aligned for cache line allocation is better */
    258 void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc);
    259 void XzUnpacker_Init(CXzUnpacker *p);
    260 void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize);
    261 void XzUnpacker_Free(CXzUnpacker *p);
    262 
    263 /*
    264   XzUnpacker
    265   The sequence for decoding functions:
    266   {
    267     XzUnpacker_Construct()
    268     [Decoding_Calls]
    269     XzUnpacker_Free()
    270   }
    271 
    272   [Decoding_Calls]
    273 
    274   There are 3 types of interfaces for [Decoding_Calls] calls:
    275 
    276   Interface-1 : Partial output buffers:
    277     {
    278       XzUnpacker_Init()
    279       for()
    280         XzUnpacker_Code();
    281     }
    282 
    283   Interface-2 : Direct output buffer:
    284     Use it, if you know exact size of decoded data, and you need
    285     whole xz unpacked data in one output buffer.
    286     xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode.
    287     {
    288       XzUnpacker_Init()
    289       XzUnpacker_SetOutBufMode(); // to set output buffer and size
    290       for()
    291         XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code()
    292     }
    293 
    294   Interface-3 : Direct output buffer : One call full decoding
    295     It unpacks whole input buffer to output buffer in one call.
    296     It uses Interface-2 internally.
    297     {
    298       XzUnpacker_CodeFull()
    299     }
    300 */
    301 
    302 /*
    303 finishMode:
    304   It has meaning only if the decoding reaches output limit (*destLen).
    305   CODER_FINISH_ANY - use smallest number of input bytes
    306   CODER_FINISH_END - read EndOfStream marker after decoding
    307 
    308 Returns:
    309   SZ_OK
    310     status:
    311       CODER_STATUS_NOT_FINISHED,
    312       CODER_STATUS_NEEDS_MORE_INPUT - maybe there are more xz streams,
    313                                       call XzUnpacker_IsStreamWasFinished to check that current stream was finished
    314   SZ_ERROR_MEM  - Memory allocation error
    315   SZ_ERROR_DATA - Data error
    316   SZ_ERROR_UNSUPPORTED - Unsupported method or method properties
    317   SZ_ERROR_CRC  - CRC error
    318   // SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src).
    319 
    320   SZ_ERROR_NO_ARCHIVE - the error with xz Stream Header with one of the following reasons:
    321      - xz Stream Signature failure
    322      - CRC32 of xz Stream Header is failed
    323      - The size of Stream padding is not multiple of four bytes.
    324     It's possible to get that error, if xz stream was finished and the stream
    325     contains some another data. In that case you can call XzUnpacker_GetExtraSize()
    326     function to get real size of xz stream.
    327 */
    328 
    329 
    330 SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
    331     const Byte *src, SizeT *srcLen, int srcFinished,
    332     ECoderFinishMode finishMode, ECoderStatus *status);
    333 
    334 SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen,
    335     const Byte *src, SizeT *srcLen,
    336     ECoderFinishMode finishMode, ECoderStatus *status);
    337 
    338 BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p);
    339 
    340 /*
    341 XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes,
    342  if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state.
    343 These bytes can be some bytes after xz archive, or
    344 it can be start of new xz stream.
    345 
    346 Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of
    347 xz stream in two cases, if XzUnpacker_Code() returns:
    348   res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT
    349   res == SZ_ERROR_NO_ARCHIVE
    350 */
    351 
    352 UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p);
    353 
    354 
    355 /*
    356   for random block decoding:
    357     XzUnpacker_Init();
    358     set CXzUnpacker::streamFlags
    359     XzUnpacker_PrepareToRandomBlockDecoding()
    360     loop
    361     {
    362       XzUnpacker_Code()
    363       XzUnpacker_IsBlockFinished()
    364     }
    365 */
    366 
    367 void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p);
    368 BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p);
    369 
    370 #define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags))
    371 
    372 
    373 
    374 /* ---------- Multi Threading Decoding ---------- */
    375 
    376 
    377 typedef struct
    378 {
    379   size_t inBufSize_ST;
    380   size_t outStep_ST;
    381   BoolInt ignoreErrors;
    382 
    383   #ifndef _7ZIP_ST
    384   unsigned numThreads;
    385   size_t inBufSize_MT;
    386   size_t memUseMax;
    387   #endif
    388 } CXzDecMtProps;
    389 
    390 void XzDecMtProps_Init(CXzDecMtProps *p);
    391 
    392 
    393 typedef void * CXzDecMtHandle;
    394 
    395 /*
    396   alloc    : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc).
    397   allocMid : for big allocations, aligned allocation is better
    398 */
    399 
    400 CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid);
    401 void XzDecMt_Destroy(CXzDecMtHandle p);
    402 
    403 
    404 typedef struct
    405 {
    406   Byte UnpackSize_Defined;
    407   Byte NumStreams_Defined;
    408   Byte NumBlocks_Defined;
    409 
    410   Byte DataAfterEnd;
    411   Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data
    412 
    413   UInt64 InSize;  // pack size processed
    414   UInt64 OutSize;
    415 
    416   UInt64 NumStreams;
    417   UInt64 NumBlocks;
    418 
    419   SRes DecodeRes;
    420   SRes ReadRes;
    421   SRes ProgressRes;
    422   SRes CombinedRes;
    423   SRes CombinedRes_Type;
    424 
    425 } CXzStatInfo;
    426 
    427 void XzStatInfo_Clear(CXzStatInfo *p);
    428 
    429 /*
    430 XzDecMt_Decode()
    431 SRes:
    432   SZ_OK               - OK
    433   SZ_ERROR_MEM        - Memory allocation error
    434   SZ_ERROR_NO_ARCHIVE - is not xz archive
    435   SZ_ERROR_ARCHIVE    - Headers error
    436   SZ_ERROR_DATA       - Data Error
    437   SZ_ERROR_CRC        - CRC Error
    438   SZ_ERROR_INPUT_EOF  - it needs more input data
    439   SZ_ERROR_WRITE      - ISeqOutStream error
    440   (SZ_ERROR_READ)     - ISeqInStream errors
    441   (SZ_ERROR_PROGRESS) - ICompressProgress errors
    442   // SZ_ERROR_THREAD     - error in multi-threading functions
    443   MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function
    444 */
    445 
    446 SRes XzDecMt_Decode(CXzDecMtHandle p,
    447     const CXzDecMtProps *props,
    448     const UInt64 *outDataSize, // NULL means undefined
    449     int finishMode,            // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished
    450     ISeqOutStream *outStream,
    451     // Byte *outBuf, size_t *outBufSize,
    452     ISeqInStream *inStream,
    453     // const Byte *inData, size_t inDataSize,
    454     CXzStatInfo *stat,
    455     int *isMT,                 // 0 means that ST (Single-Thread) version was used
    456     ICompressProgress *progress);
    457 
    458 EXTERN_C_END
    459 
    460 #endif
    461