1 // 7zEncode.cpp 2 3 #include "StdAfx.h" 4 5 #include "../../Common/CreateCoder.h" 6 #include "../../Common/FilterCoder.h" 7 #include "../../Common/LimitedStreams.h" 8 #include "../../Common/InOutTempBuffer.h" 9 #include "../../Common/ProgressUtils.h" 10 #include "../../Common/StreamObjects.h" 11 12 #include "7zEncode.h" 13 #include "7zSpecStream.h" 14 15 namespace NArchive { 16 namespace N7z { 17 18 void CEncoder::InitBindConv() 19 { 20 unsigned numIn = _bindInfo.Coders.Size(); 21 22 _SrcIn_to_DestOut.ClearAndSetSize(numIn); 23 _DestOut_to_SrcIn.ClearAndSetSize(numIn); 24 25 unsigned numOut = _bindInfo.GetNum_Bonds_and_PackStreams(); 26 _SrcOut_to_DestIn.ClearAndSetSize(numOut); 27 // _DestIn_to_SrcOut.ClearAndSetSize(numOut); 28 29 UInt32 destIn = 0; 30 UInt32 destOut = 0; 31 32 for (unsigned i = _bindInfo.Coders.Size(); i != 0;) 33 { 34 i--; 35 36 const NCoderMixer2::CCoderStreamsInfo &coder = _bindInfo.Coders[i]; 37 38 numIn--; 39 numOut -= coder.NumStreams; 40 41 _SrcIn_to_DestOut[numIn] = destOut; 42 _DestOut_to_SrcIn[destOut] = numIn; 43 44 destOut++; 45 46 for (UInt32 j = 0; j < coder.NumStreams; j++, destIn++) 47 { 48 UInt32 index = numOut + j; 49 _SrcOut_to_DestIn[index] = destIn; 50 // _DestIn_to_SrcOut[destIn] = index; 51 } 52 } 53 } 54 55 void CEncoder::SetFolder(CFolder &folder) 56 { 57 folder.Bonds.SetSize(_bindInfo.Bonds.Size()); 58 59 unsigned i; 60 61 for (i = 0; i < _bindInfo.Bonds.Size(); i++) 62 { 63 CBond &fb = folder.Bonds[i]; 64 const NCoderMixer2::CBond &mixerBond = _bindInfo.Bonds[_bindInfo.Bonds.Size() - 1 - i]; 65 fb.PackIndex = _SrcOut_to_DestIn[mixerBond.PackIndex]; 66 fb.UnpackIndex = _SrcIn_to_DestOut[mixerBond.UnpackIndex]; 67 } 68 69 folder.Coders.SetSize(_bindInfo.Coders.Size()); 70 71 for (i = 0; i < _bindInfo.Coders.Size(); i++) 72 { 73 CCoderInfo &coderInfo = folder.Coders[i]; 74 const NCoderMixer2::CCoderStreamsInfo &coderStreamsInfo = _bindInfo.Coders[_bindInfo.Coders.Size() - 1 - i]; 75 76 coderInfo.NumStreams = coderStreamsInfo.NumStreams; 77 coderInfo.MethodID = _decompressionMethods[i]; 78 // we don't free coderInfo.Props here. So coderInfo.Props can be non-empty. 79 } 80 81 folder.PackStreams.SetSize(_bindInfo.PackStreams.Size()); 82 83 for (i = 0; i < _bindInfo.PackStreams.Size(); i++) 84 folder.PackStreams[i] = _SrcOut_to_DestIn[_bindInfo.PackStreams[i]]; 85 } 86 87 88 89 static HRESULT SetCoderProps2(const CProps &props, const UInt64 *dataSizeReduce, IUnknown *coder) 90 { 91 CMyComPtr<ICompressSetCoderProperties> setCoderProperties; 92 coder->QueryInterface(IID_ICompressSetCoderProperties, (void **)&setCoderProperties); 93 if (setCoderProperties) 94 return props.SetCoderProps(setCoderProperties, dataSizeReduce); 95 return props.AreThereNonOptionalProps() ? E_INVALIDARG : S_OK; 96 } 97 98 99 100 void CMtEncMultiProgress::Init(ICompressProgressInfo *progress) 101 { 102 _progress = progress; 103 OutSize = 0; 104 } 105 106 STDMETHODIMP CMtEncMultiProgress::SetRatioInfo(const UInt64 *inSize, const UInt64 * /* outSize */) 107 { 108 UInt64 outSize2; 109 { 110 #ifndef _7ZIP_ST 111 NWindows::NSynchronization::CCriticalSectionLock lock(CriticalSection); 112 #endif 113 outSize2 = OutSize; 114 } 115 116 if (_progress) 117 return _progress->SetRatioInfo(inSize, &outSize2); 118 119 return S_OK; 120 } 121 122 123 124 HRESULT CEncoder::CreateMixerCoder( 125 DECL_EXTERNAL_CODECS_LOC_VARS 126 const UInt64 *inSizeForReduce) 127 { 128 #ifdef USE_MIXER_MT 129 #ifdef USE_MIXER_ST 130 if (_options.MultiThreadMixer) 131 #endif 132 { 133 _mixerMT = new NCoderMixer2::CMixerMT(true); 134 _mixerRef = _mixerMT; 135 _mixer = _mixerMT; 136 } 137 #ifdef USE_MIXER_ST 138 else 139 #endif 140 #endif 141 { 142 #ifdef USE_MIXER_ST 143 _mixerST = new NCoderMixer2::CMixerST(true); 144 _mixerRef = _mixerST; 145 _mixer = _mixerST; 146 #endif 147 } 148 149 RINOK(_mixer->SetBindInfo(_bindInfo)); 150 151 FOR_VECTOR (m, _options.Methods) 152 { 153 const CMethodFull &methodFull = _options.Methods[m]; 154 155 CCreatedCoder cod; 156 157 RINOK(CreateCoder( 158 EXTERNAL_CODECS_LOC_VARS 159 methodFull.Id, true, cod)); 160 161 if (cod.NumStreams != methodFull.NumStreams) 162 return E_FAIL; 163 if (!cod.Coder && !cod.Coder2) 164 return E_FAIL; 165 166 CMyComPtr<IUnknown> encoderCommon = cod.Coder ? (IUnknown *)cod.Coder : (IUnknown *)cod.Coder2; 167 168 #ifndef _7ZIP_ST 169 { 170 CMyComPtr<ICompressSetCoderMt> setCoderMt; 171 encoderCommon.QueryInterface(IID_ICompressSetCoderMt, &setCoderMt); 172 if (setCoderMt) 173 { 174 RINOK(setCoderMt->SetNumberOfThreads(_options.NumThreads)); 175 } 176 } 177 #endif 178 179 RINOK(SetCoderProps2(methodFull, inSizeForReduce, encoderCommon)); 180 181 /* 182 CMyComPtr<ICryptoResetSalt> resetSalt; 183 encoderCommon.QueryInterface(IID_ICryptoResetSalt, (void **)&resetSalt); 184 if (resetSalt) 185 { 186 resetSalt->ResetSalt(); 187 } 188 */ 189 190 // now there is no codec that uses another external codec 191 /* 192 #ifdef EXTERNAL_CODECS 193 CMyComPtr<ISetCompressCodecsInfo> setCompressCodecsInfo; 194 encoderCommon.QueryInterface(IID_ISetCompressCodecsInfo, (void **)&setCompressCodecsInfo); 195 if (setCompressCodecsInfo) 196 { 197 // we must use g_ExternalCodecs also 198 RINOK(setCompressCodecsInfo->SetCompressCodecsInfo(__externalCodecs->GetCodecs)); 199 } 200 #endif 201 */ 202 203 CMyComPtr<ICryptoSetPassword> cryptoSetPassword; 204 encoderCommon.QueryInterface(IID_ICryptoSetPassword, &cryptoSetPassword); 205 206 if (cryptoSetPassword) 207 { 208 const unsigned sizeInBytes = _options.Password.Len() * 2; 209 CByteBuffer buffer(sizeInBytes); 210 for (unsigned i = 0; i < _options.Password.Len(); i++) 211 { 212 wchar_t c = _options.Password[i]; 213 ((Byte *)buffer)[i * 2] = (Byte)c; 214 ((Byte *)buffer)[i * 2 + 1] = (Byte)(c >> 8); 215 } 216 RINOK(cryptoSetPassword->CryptoSetPassword((const Byte *)buffer, (UInt32)sizeInBytes)); 217 } 218 219 _mixer->AddCoder(cod); 220 } 221 return S_OK; 222 } 223 224 225 226 class CSequentialOutTempBufferImp2: 227 public ISequentialOutStream, 228 public CMyUnknownImp 229 { 230 CInOutTempBuffer *_buf; 231 public: 232 CMtEncMultiProgress *_mtProgresSpec; 233 234 CSequentialOutTempBufferImp2(): _buf(0), _mtProgresSpec(NULL) {} 235 void Init(CInOutTempBuffer *buffer) { _buf = buffer; } 236 MY_UNKNOWN_IMP1(ISequentialOutStream) 237 238 STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); 239 }; 240 241 STDMETHODIMP CSequentialOutTempBufferImp2::Write(const void *data, UInt32 size, UInt32 *processed) 242 { 243 if (!_buf->Write(data, size)) 244 { 245 if (processed) 246 *processed = 0; 247 return E_FAIL; 248 } 249 if (processed) 250 *processed = size; 251 if (_mtProgresSpec) 252 _mtProgresSpec->AddOutSize(size); 253 return S_OK; 254 } 255 256 257 class CSequentialOutMtNotify: 258 public ISequentialOutStream, 259 public CMyUnknownImp 260 { 261 public: 262 CMyComPtr<ISequentialOutStream> _stream; 263 CMtEncMultiProgress *_mtProgresSpec; 264 265 CSequentialOutMtNotify(): _mtProgresSpec(NULL) {} 266 MY_UNKNOWN_IMP1(ISequentialOutStream) 267 268 STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); 269 }; 270 271 STDMETHODIMP CSequentialOutMtNotify::Write(const void *data, UInt32 size, UInt32 *processed) 272 { 273 UInt32 realProcessed = 0; 274 HRESULT res = _stream->Write(data, size, &realProcessed); 275 if (processed) 276 *processed = realProcessed; 277 if (_mtProgresSpec) 278 _mtProgresSpec->AddOutSize(size); 279 return res; 280 } 281 282 283 284 HRESULT CEncoder::Encode( 285 DECL_EXTERNAL_CODECS_LOC_VARS 286 ISequentialInStream *inStream, 287 // const UInt64 *inStreamSize, 288 const UInt64 *inSizeForReduce, 289 CFolder &folderItem, 290 CRecordVector<UInt64> &coderUnpackSizes, 291 UInt64 &unpackSize, 292 ISequentialOutStream *outStream, 293 CRecordVector<UInt64> &packSizes, 294 ICompressProgressInfo *compressProgress) 295 { 296 RINOK(EncoderConstr()); 297 298 if (!_mixerRef) 299 { 300 RINOK(CreateMixerCoder(EXTERNAL_CODECS_LOC_VARS inSizeForReduce)); 301 } 302 303 _mixer->ReInit(); 304 305 CMtEncMultiProgress *mtProgressSpec = NULL; 306 CMyComPtr<ICompressProgressInfo> mtProgress; 307 308 CSequentialOutMtNotify *mtOutStreamNotifySpec = NULL; 309 CMyComPtr<ISequentialOutStream> mtOutStreamNotify; 310 311 CObjectVector<CInOutTempBuffer> inOutTempBuffers; 312 CObjectVector<CSequentialOutTempBufferImp2 *> tempBufferSpecs; 313 CObjectVector<CMyComPtr<ISequentialOutStream> > tempBuffers; 314 315 unsigned numMethods = _bindInfo.Coders.Size(); 316 317 unsigned i; 318 319 for (i = 1; i < _bindInfo.PackStreams.Size(); i++) 320 { 321 CInOutTempBuffer &iotb = inOutTempBuffers.AddNew(); 322 iotb.Create(); 323 iotb.InitWriting(); 324 } 325 326 for (i = 1; i < _bindInfo.PackStreams.Size(); i++) 327 { 328 CSequentialOutTempBufferImp2 *tempBufferSpec = new CSequentialOutTempBufferImp2; 329 CMyComPtr<ISequentialOutStream> tempBuffer = tempBufferSpec; 330 tempBufferSpec->Init(&inOutTempBuffers[i - 1]); 331 tempBuffers.Add(tempBuffer); 332 tempBufferSpecs.Add(tempBufferSpec); 333 } 334 335 for (i = 0; i < numMethods; i++) 336 _mixer->SetCoderInfo(i, NULL, NULL); 337 338 339 /* inStreamSize can be used by BCJ2 to set optimal range of conversion. 340 But current BCJ2 encoder uses also another way to check exact size of current file. 341 So inStreamSize is not required. */ 342 343 /* 344 if (inStreamSize) 345 _mixer->SetCoderInfo(_bindInfo.UnpackCoder, inStreamSize, NULL); 346 */ 347 348 349 CSequentialInStreamSizeCount2 *inStreamSizeCountSpec = new CSequentialInStreamSizeCount2; 350 CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec; 351 352 CSequentialOutStreamSizeCount *outStreamSizeCountSpec = NULL; 353 CMyComPtr<ISequentialOutStream> outStreamSizeCount; 354 355 inStreamSizeCountSpec->Init(inStream); 356 357 ISequentialInStream *inStreamPointer = inStreamSizeCount; 358 CRecordVector<ISequentialOutStream *> outStreamPointers; 359 360 SetFolder(folderItem); 361 362 for (i = 0; i < numMethods; i++) 363 { 364 IUnknown *coder = _mixer->GetCoder(i).GetUnknown(); 365 366 CMyComPtr<ICryptoResetInitVector> resetInitVector; 367 coder->QueryInterface(IID_ICryptoResetInitVector, (void **)&resetInitVector); 368 if (resetInitVector) 369 { 370 resetInitVector->ResetInitVector(); 371 } 372 373 CMyComPtr<ICompressWriteCoderProperties> writeCoderProperties; 374 coder->QueryInterface(IID_ICompressWriteCoderProperties, (void **)&writeCoderProperties); 375 376 CByteBuffer &props = folderItem.Coders[numMethods - 1 - i].Props; 377 378 if (writeCoderProperties) 379 { 380 CDynBufSeqOutStream *outStreamSpec = new CDynBufSeqOutStream; 381 CMyComPtr<ISequentialOutStream> dynOutStream(outStreamSpec); 382 outStreamSpec->Init(); 383 writeCoderProperties->WriteCoderProperties(dynOutStream); 384 outStreamSpec->CopyToBuffer(props); 385 } 386 else 387 props.Free(); 388 } 389 390 _mixer->SelectMainCoder(false); 391 UInt32 mainCoder = _mixer->MainCoderIndex; 392 393 bool useMtProgress = false; 394 if (!_mixer->Is_PackSize_Correct_for_Coder(mainCoder)) 395 { 396 #ifdef _7ZIP_ST 397 if (!_mixer->IsThere_ExternalCoder_in_PackTree(mainCoder)) 398 #endif 399 useMtProgress = true; 400 } 401 402 if (useMtProgress) 403 { 404 mtProgressSpec = new CMtEncMultiProgress; 405 mtProgress = mtProgressSpec; 406 mtProgressSpec->Init(compressProgress); 407 408 mtOutStreamNotifySpec = new CSequentialOutMtNotify; 409 mtOutStreamNotify = mtOutStreamNotifySpec; 410 mtOutStreamNotifySpec->_stream = outStream; 411 mtOutStreamNotifySpec->_mtProgresSpec = mtProgressSpec; 412 413 FOR_VECTOR(t, tempBufferSpecs) 414 { 415 tempBufferSpecs[t]->_mtProgresSpec = mtProgressSpec; 416 } 417 } 418 419 420 if (_bindInfo.PackStreams.Size() != 0) 421 { 422 outStreamSizeCountSpec = new CSequentialOutStreamSizeCount; 423 outStreamSizeCount = outStreamSizeCountSpec; 424 outStreamSizeCountSpec->SetStream(mtOutStreamNotify ? (ISequentialOutStream *)mtOutStreamNotify : outStream); 425 outStreamSizeCountSpec->Init(); 426 outStreamPointers.Add(outStreamSizeCount); 427 } 428 429 for (i = 1; i < _bindInfo.PackStreams.Size(); i++) 430 outStreamPointers.Add(tempBuffers[i - 1]); 431 432 RINOK(_mixer->Code( 433 &inStreamPointer, 434 &outStreamPointers.Front(), 435 mtProgress ? (ICompressProgressInfo *)mtProgress : compressProgress)); 436 437 if (_bindInfo.PackStreams.Size() != 0) 438 packSizes.Add(outStreamSizeCountSpec->GetSize()); 439 440 for (i = 1; i < _bindInfo.PackStreams.Size(); i++) 441 { 442 CInOutTempBuffer &inOutTempBuffer = inOutTempBuffers[i - 1]; 443 RINOK(inOutTempBuffer.WriteToStream(outStream)); 444 packSizes.Add(inOutTempBuffer.GetDataSize()); 445 } 446 447 unpackSize = 0; 448 449 for (i = 0; i < _bindInfo.Coders.Size(); i++) 450 { 451 int bond = _bindInfo.FindBond_for_UnpackStream(_DestOut_to_SrcIn[i]); 452 UInt64 streamSize; 453 if (bond < 0) 454 { 455 streamSize = inStreamSizeCountSpec->GetSize(); 456 unpackSize = streamSize; 457 } 458 else 459 streamSize = _mixer->GetBondStreamSize(bond); 460 coderUnpackSizes.Add(streamSize); 461 } 462 463 return S_OK; 464 } 465 466 467 CEncoder::CEncoder(const CCompressionMethodMode &options): 468 _constructed(false) 469 { 470 if (options.IsEmpty()) 471 throw 1; 472 473 _options = options; 474 475 #ifdef USE_MIXER_ST 476 _mixerST = NULL; 477 #endif 478 479 #ifdef USE_MIXER_MT 480 _mixerMT = NULL; 481 #endif 482 483 _mixer = NULL; 484 } 485 486 487 HRESULT CEncoder::EncoderConstr() 488 { 489 if (_constructed) 490 return S_OK; 491 if (_options.Methods.IsEmpty()) 492 { 493 // it has only password method; 494 if (!_options.PasswordIsDefined) 495 throw 1; 496 if (!_options.Bonds.IsEmpty()) 497 throw 1; 498 499 CMethodFull method; 500 method.Id = k_AES; 501 method.NumStreams = 1; 502 _options.Methods.Add(method); 503 504 NCoderMixer2::CCoderStreamsInfo coderStreamsInfo; 505 coderStreamsInfo.NumStreams = 1; 506 _bindInfo.Coders.Add(coderStreamsInfo); 507 508 _bindInfo.PackStreams.Add(0); 509 _bindInfo.UnpackCoder = 0; 510 } 511 else 512 { 513 514 UInt32 numOutStreams = 0; 515 unsigned i; 516 517 for (i = 0; i < _options.Methods.Size(); i++) 518 { 519 const CMethodFull &methodFull = _options.Methods[i]; 520 NCoderMixer2::CCoderStreamsInfo cod; 521 522 cod.NumStreams = methodFull.NumStreams; 523 524 if (_options.Bonds.IsEmpty()) 525 { 526 // if there are no bonds in options, we create bonds via first streams of coders 527 if (i != _options.Methods.Size() - 1) 528 { 529 NCoderMixer2::CBond bond; 530 bond.PackIndex = numOutStreams; 531 bond.UnpackIndex = i + 1; // it's next coder 532 _bindInfo.Bonds.Add(bond); 533 } 534 else if (cod.NumStreams != 0) 535 _bindInfo.PackStreams.Insert(0, numOutStreams); 536 537 for (UInt32 j = 1; j < cod.NumStreams; j++) 538 _bindInfo.PackStreams.Add(numOutStreams + j); 539 } 540 541 numOutStreams += cod.NumStreams; 542 543 _bindInfo.Coders.Add(cod); 544 } 545 546 if (!_options.Bonds.IsEmpty()) 547 { 548 for (i = 0; i < _options.Bonds.Size(); i++) 549 { 550 NCoderMixer2::CBond mixerBond; 551 const CBond2 &bond = _options.Bonds[i]; 552 if (bond.InCoder >= _bindInfo.Coders.Size() 553 || bond.OutCoder >= _bindInfo.Coders.Size() 554 || bond.OutStream >= _bindInfo.Coders[bond.OutCoder].NumStreams) 555 return E_INVALIDARG; 556 mixerBond.PackIndex = _bindInfo.GetStream_for_Coder(bond.OutCoder) + bond.OutStream; 557 mixerBond.UnpackIndex = bond.InCoder; 558 _bindInfo.Bonds.Add(mixerBond); 559 } 560 561 for (i = 0; i < numOutStreams; i++) 562 if (_bindInfo.FindBond_for_PackStream(i) == -1) 563 _bindInfo.PackStreams.Add(i); 564 } 565 566 if (!_bindInfo.SetUnpackCoder()) 567 return E_INVALIDARG; 568 569 if (!_bindInfo.CalcMapsAndCheck()) 570 return E_INVALIDARG; 571 572 if (_bindInfo.PackStreams.Size() != 1) 573 { 574 /* main_PackStream is pack stream of main path of coders tree. 575 We find main_PackStream, and place to start of list of out streams. 576 It allows to use more optimal memory usage for temp buffers, 577 if main_PackStream is largest stream. */ 578 579 UInt32 ci = _bindInfo.UnpackCoder; 580 581 for (;;) 582 { 583 if (_bindInfo.Coders[ci].NumStreams == 0) 584 break; 585 586 UInt32 outIndex = _bindInfo.Coder_to_Stream[ci]; 587 int bond = _bindInfo.FindBond_for_PackStream(outIndex); 588 if (bond >= 0) 589 { 590 ci = _bindInfo.Bonds[bond].UnpackIndex; 591 continue; 592 } 593 594 int si = _bindInfo.FindStream_in_PackStreams(outIndex); 595 if (si >= 0) 596 _bindInfo.PackStreams.MoveToFront(si); 597 break; 598 } 599 } 600 601 if (_options.PasswordIsDefined) 602 { 603 unsigned numCryptoStreams = _bindInfo.PackStreams.Size(); 604 605 unsigned numInStreams = _bindInfo.Coders.Size(); 606 607 for (i = 0; i < numCryptoStreams; i++) 608 { 609 NCoderMixer2::CBond bond; 610 bond.UnpackIndex = numInStreams + i; 611 bond.PackIndex = _bindInfo.PackStreams[i]; 612 _bindInfo.Bonds.Add(bond); 613 } 614 _bindInfo.PackStreams.Clear(); 615 616 /* 617 if (numCryptoStreams == 0) 618 numCryptoStreams = 1; 619 */ 620 621 for (i = 0; i < numCryptoStreams; i++) 622 { 623 CMethodFull method; 624 method.NumStreams = 1; 625 method.Id = k_AES; 626 _options.Methods.Add(method); 627 628 NCoderMixer2::CCoderStreamsInfo cod; 629 cod.NumStreams = 1; 630 _bindInfo.Coders.Add(cod); 631 632 _bindInfo.PackStreams.Add(numOutStreams++); 633 } 634 } 635 636 } 637 638 for (unsigned i = _options.Methods.Size(); i != 0;) 639 _decompressionMethods.Add(_options.Methods[--i].Id); 640 641 if (_bindInfo.Coders.Size() > 16) 642 return E_INVALIDARG; 643 if (_bindInfo.GetNum_Bonds_and_PackStreams() > 16) 644 return E_INVALIDARG; 645 646 if (!_bindInfo.CalcMapsAndCheck()) 647 return E_INVALIDARG; 648 649 InitBindConv(); 650 _constructed = true; 651 return S_OK; 652 } 653 654 CEncoder::~CEncoder() {} 655 656 }} 657