1 // 7zUpdate.cpp 2 3 #include "StdAfx.h" 4 5 #include "../../../../C/CpuArch.h" 6 7 #include "../../../Common/Wildcard.h" 8 9 #include "../../Common/CreateCoder.h" 10 #include "../../Common/LimitedStreams.h" 11 #include "../../Common/ProgressUtils.h" 12 13 #include "../../Compress/CopyCoder.h" 14 15 #include "../Common/ItemNameUtils.h" 16 17 #include "7zDecode.h" 18 #include "7zEncode.h" 19 #include "7zFolderInStream.h" 20 #include "7zHandler.h" 21 #include "7zOut.h" 22 #include "7zUpdate.h" 23 24 namespace NArchive { 25 namespace N7z { 26 27 28 #define k_X86 k_BCJ 29 30 struct CFilterMode 31 { 32 UInt32 Id; 33 UInt32 Delta; 34 35 CFilterMode(): Id(0), Delta(0) {} 36 37 void SetDelta() 38 { 39 if (Id == k_IA64) 40 Delta = 16; 41 else if (Id == k_ARM || Id == k_PPC || Id == k_SPARC) 42 Delta = 4; 43 else if (Id == k_ARMT) 44 Delta = 2; 45 else 46 Delta = 0; 47 } 48 }; 49 50 51 /* ---------- PE ---------- */ 52 53 #define MZ_SIG 0x5A4D 54 55 #define PE_SIG 0x00004550 56 #define PE_OptHeader_Magic_32 0x10B 57 #define PE_OptHeader_Magic_64 0x20B 58 #define PE_SectHeaderSize 40 59 #define PE_SECT_EXECUTE 0x20000000 60 61 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) 62 { 63 if (size < 512 || GetUi16(buf) != MZ_SIG) 64 return 0; 65 66 const Byte *p; 67 UInt32 peOffset, optHeaderSize, filterId; 68 69 peOffset = GetUi32(buf + 0x3C); 70 if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0) 71 return 0; 72 p = buf + peOffset; 73 if (GetUi32(p) != PE_SIG) 74 return 0; 75 p += 4; 76 77 switch (GetUi16(p)) 78 { 79 case 0x014C: 80 case 0x8664: filterId = k_X86; break; 81 82 /* 83 IMAGE_FILE_MACHINE_ARM 0x01C0 // ARM LE 84 IMAGE_FILE_MACHINE_THUMB 0x01C2 // ARM Thumb / Thumb-2 LE 85 IMAGE_FILE_MACHINE_ARMNT 0x01C4 // ARM Thumb-2, LE 86 Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2). 87 */ 88 89 case 0x01C0: // WinCE old 90 case 0x01C2: filterId = k_ARM; break; // WinCE new 91 case 0x01C4: filterId = k_ARMT; break; // WinRT 92 93 case 0x0200: filterId = k_IA64; break; 94 default: return 0; 95 } 96 97 optHeaderSize = GetUi16(p + 16); 98 if (optHeaderSize > (1 << 10)) 99 return 0; 100 101 p += 20; /* headerSize */ 102 103 switch (GetUi16(p)) 104 { 105 case PE_OptHeader_Magic_32: 106 case PE_OptHeader_Magic_64: 107 break; 108 default: 109 return 0; 110 } 111 112 filterMode->Id = filterId; 113 return 1; 114 } 115 116 117 /* ---------- ELF ---------- */ 118 119 #define ELF_SIG 0x464C457F 120 121 #define ELF_CLASS_32 1 122 #define ELF_CLASS_64 2 123 124 #define ELF_DATA_2LSB 1 125 #define ELF_DATA_2MSB 2 126 127 static UInt16 Get16(const Byte *p, BoolInt be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); } 128 static UInt32 Get32(const Byte *p, BoolInt be) { if (be) return GetBe32(p); return GetUi32(p); } 129 // static UInt64 Get64(const Byte *p, BoolInt be) { if (be) return GetBe64(p); return GetUi64(p); } 130 131 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode) 132 { 133 BoolInt /* is32, */ be; 134 UInt32 filterId; 135 136 if (size < 512 || buf[6] != 1) /* ver */ 137 return 0; 138 139 if (GetUi32(buf) != ELF_SIG) 140 return 0; 141 142 switch (buf[4]) 143 { 144 case ELF_CLASS_32: /* is32 = True; */ break; 145 case ELF_CLASS_64: /* is32 = False; */ break; 146 default: return 0; 147 } 148 149 switch (buf[5]) 150 { 151 case ELF_DATA_2LSB: be = False; break; 152 case ELF_DATA_2MSB: be = True; break; 153 default: return 0; 154 } 155 156 switch (Get16(buf + 0x12, be)) 157 { 158 case 3: 159 case 6: 160 case 62: filterId = k_X86; break; 161 case 2: 162 case 18: 163 case 43: filterId = k_SPARC; break; 164 case 20: 165 case 21: if (!be) return 0; filterId = k_PPC; break; 166 case 40: if ( be) return 0; filterId = k_ARM; break; 167 168 /* Some IA-64 ELF exacutable have size that is not aligned for 16 bytes. 169 So we don't use IA-64 filter for IA-64 ELF */ 170 // case 50: if ( be) return 0; filterId = k_IA64; break; 171 172 default: return 0; 173 } 174 175 filterMode->Id = filterId; 176 return 1; 177 } 178 179 180 181 /* ---------- Mach-O ---------- */ 182 183 #define MACH_SIG_BE_32 0xCEFAEDFE 184 #define MACH_SIG_BE_64 0xCFFAEDFE 185 #define MACH_SIG_LE_32 0xFEEDFACE 186 #define MACH_SIG_LE_64 0xFEEDFACF 187 188 #define MACH_ARCH_ABI64 (1 << 24) 189 #define MACH_MACHINE_386 7 190 #define MACH_MACHINE_ARM 12 191 #define MACH_MACHINE_SPARC 14 192 #define MACH_MACHINE_PPC 18 193 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC) 194 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386) 195 196 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode) 197 { 198 UInt32 filterId, numCommands, commandsSize; 199 200 if (size < 512) 201 return 0; 202 203 BoolInt /* mode64, */ be; 204 switch (GetUi32(buf)) 205 { 206 case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break; 207 case MACH_SIG_BE_64: /* mode64 = True; */ be = True; break; 208 case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break; 209 case MACH_SIG_LE_64: /* mode64 = True; */ be = False; break; 210 default: return 0; 211 } 212 213 switch (Get32(buf + 4, be)) 214 { 215 case MACH_MACHINE_386: 216 case MACH_MACHINE_AMD64: filterId = k_X86; break; 217 case MACH_MACHINE_ARM: if ( be) return 0; filterId = k_ARM; break; 218 case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break; 219 case MACH_MACHINE_PPC: 220 case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break; 221 default: return 0; 222 } 223 224 numCommands = Get32(buf + 0x10, be); 225 commandsSize = Get32(buf + 0x14, be); 226 227 if (commandsSize > (1 << 24) || numCommands > (1 << 18)) 228 return 0; 229 230 filterMode->Id = filterId; 231 return 1; 232 } 233 234 235 /* ---------- WAV ---------- */ 236 237 #define WAV_SUBCHUNK_fmt 0x20746D66 238 #define WAV_SUBCHUNK_data 0x61746164 239 240 #define RIFF_SIG 0x46464952 241 242 static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) 243 { 244 UInt32 subChunkSize, pos; 245 if (size < 0x2C) 246 return False; 247 248 if (GetUi32(buf + 0) != RIFF_SIG || 249 GetUi32(buf + 8) != 0x45564157 || // WAVE 250 GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt) 251 return False; 252 subChunkSize = GetUi32(buf + 0x10); 253 /* [0x14 = format] = 1 (PCM) */ 254 if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1) 255 return False; 256 257 unsigned numChannels = GetUi16(buf + 0x16); 258 unsigned bitsPerSample = GetUi16(buf + 0x22); 259 260 if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256) 261 return False; 262 263 pos = 0x14 + subChunkSize; 264 265 const int kNumSubChunksTests = 10; 266 // Do we need to scan more than 3 sub-chunks? 267 for (int i = 0; i < kNumSubChunksTests; i++) 268 { 269 if (pos + 8 > size) 270 return False; 271 subChunkSize = GetUi32(buf + pos + 4); 272 if (GetUi32(buf + pos) == WAV_SUBCHUNK_data) 273 { 274 unsigned delta = numChannels * (bitsPerSample >> 3); 275 if (delta >= 256) 276 return False; 277 filterMode->Id = k_Delta; 278 filterMode->Delta = delta; 279 return True; 280 } 281 if (subChunkSize > (1 << 16)) 282 return False; 283 pos += subChunkSize + 8; 284 } 285 return False; 286 } 287 288 static BoolInt ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode) 289 { 290 filterMode->Id = 0; 291 filterMode->Delta = 0; 292 293 if (Parse_EXE(buf, size, filterMode)) return True; 294 if (Parse_ELF(buf, size, filterMode)) return True; 295 if (Parse_MACH(buf, size, filterMode)) return True; 296 return Parse_WAV(buf, size, filterMode); 297 } 298 299 300 301 302 struct CFilterMode2: public CFilterMode 303 { 304 bool Encrypted; 305 unsigned GroupIndex; 306 307 CFilterMode2(): Encrypted(false) {} 308 309 int Compare(const CFilterMode2 &m) const 310 { 311 if (!Encrypted) 312 { 313 if (m.Encrypted) 314 return -1; 315 } 316 else if (!m.Encrypted) 317 return 1; 318 319 if (Id < m.Id) return -1; 320 if (Id > m.Id) return 1; 321 322 if (Delta < m.Delta) return -1; 323 if (Delta > m.Delta) return 1; 324 325 return 0; 326 } 327 328 bool operator ==(const CFilterMode2 &m) const 329 { 330 return Id == m.Id && Delta == m.Delta && Encrypted == m.Encrypted; 331 } 332 }; 333 334 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m) 335 { 336 unsigned i; 337 for (i = 0; i < filters.Size(); i++) 338 { 339 const CFilterMode2 &m2 = filters[i]; 340 if (m == m2) 341 return i; 342 /* 343 if (m.Encrypted != m2.Encrypted) 344 { 345 if (!m.Encrypted) 346 break; 347 continue; 348 } 349 350 if (m.Id < m2.Id) break; 351 if (m.Id != m2.Id) continue; 352 353 if (m.Delta < m2.Delta) break; 354 if (m.Delta != m2.Delta) continue; 355 */ 356 } 357 // filters.Insert(i, m); 358 // return i; 359 return filters.Add(m); 360 } 361 362 static inline bool Is86Filter(CMethodId m) 363 { 364 return (m == k_BCJ || m == k_BCJ2); 365 } 366 367 static inline bool IsExeFilter(CMethodId m) 368 { 369 switch (m) 370 { 371 case k_BCJ: 372 case k_BCJ2: 373 case k_ARM: 374 case k_ARMT: 375 case k_PPC: 376 case k_SPARC: 377 case k_IA64: 378 return true; 379 } 380 return false; 381 } 382 383 static unsigned Get_FilterGroup_for_Folder( 384 CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter) 385 { 386 CFilterMode2 m; 387 m.Id = 0; 388 m.Delta = 0; 389 m.Encrypted = f.IsEncrypted(); 390 391 if (extractFilter) 392 { 393 const CCoderInfo &coder = f.Coders[f.UnpackCoder]; 394 395 if (coder.MethodID == k_Delta) 396 { 397 if (coder.Props.Size() == 1) 398 { 399 m.Delta = (unsigned)coder.Props[0] + 1; 400 m.Id = k_Delta; 401 } 402 } 403 else if (IsExeFilter(coder.MethodID)) 404 { 405 m.Id = (UInt32)coder.MethodID; 406 if (m.Id == k_BCJ2) 407 m.Id = k_BCJ; 408 m.SetDelta(); 409 } 410 } 411 412 return GetGroup(filters, m); 413 } 414 415 416 417 418 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream, 419 UInt64 position, UInt64 size, ICompressProgressInfo *progress) 420 { 421 RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0)); 422 CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; 423 CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec); 424 streamSpec->SetStream(inStream); 425 streamSpec->Init(size); 426 427 NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder; 428 CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec; 429 RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress)); 430 return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL); 431 } 432 433 /* 434 unsigned CUpdateItem::GetExtensionPos() const 435 { 436 int slashPos = Name.ReverseFind_PathSepar(); 437 int dotPos = Name.ReverseFind_Dot(); 438 if (dotPos <= slashPos) 439 return Name.Len(); 440 return dotPos + 1; 441 } 442 443 UString CUpdateItem::GetExtension() const 444 { 445 return Name.Ptr(GetExtensionPos()); 446 } 447 */ 448 449 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; } 450 451 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b)) 452 453 /* 454 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2) 455 { 456 size_t c1 = a1.GetCapacity(); 457 size_t c2 = a2.GetCapacity(); 458 RINOZ_COMP(c1, c2); 459 for (size_t i = 0; i < c1; i++) 460 RINOZ_COMP(a1[i], a2[i]); 461 return 0; 462 } 463 464 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2) 465 { 466 RINOZ_COMP(c1.NumInStreams, c2.NumInStreams); 467 RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams); 468 RINOZ_COMP(c1.MethodID, c2.MethodID); 469 return CompareBuffers(c1.Props, c2.Props); 470 } 471 472 static int CompareBonds(const CBond &b1, const CBond &b2) 473 { 474 RINOZ_COMP(b1.InIndex, b2.InIndex); 475 return MyCompare(b1.OutIndex, b2.OutIndex); 476 } 477 478 static int CompareFolders(const CFolder &f1, const CFolder &f2) 479 { 480 int s1 = f1.Coders.Size(); 481 int s2 = f2.Coders.Size(); 482 RINOZ_COMP(s1, s2); 483 int i; 484 for (i = 0; i < s1; i++) 485 RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i])); 486 s1 = f1.Bonds.Size(); 487 s2 = f2.Bonds.Size(); 488 RINOZ_COMP(s1, s2); 489 for (i = 0; i < s1; i++) 490 RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i])); 491 return 0; 492 } 493 */ 494 495 /* 496 static int CompareFiles(const CFileItem &f1, const CFileItem &f2) 497 { 498 return CompareFileNames(f1.Name, f2.Name); 499 } 500 */ 501 502 struct CFolderRepack 503 { 504 unsigned FolderIndex; 505 CNum NumCopyFiles; 506 }; 507 508 /* 509 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *) 510 { 511 int i1 = p1->FolderIndex; 512 int i2 = p2->FolderIndex; 513 // In that version we don't want to parse folders here, so we don't compare folders 514 // probably it must be improved in future 515 // const CDbEx &db = *(const CDbEx *)param; 516 // RINOZ(CompareFolders( 517 // db.Folders[i1], 518 // db.Folders[i2])); 519 520 return MyCompare(i1, i2); 521 522 // RINOZ_COMP( 523 // db.NumUnpackStreamsVector[i1], 524 // db.NumUnpackStreamsVector[i2]); 525 // if (db.NumUnpackStreamsVector[i1] == 0) 526 // return 0; 527 // return CompareFiles( 528 // db.Files[db.FolderStartFileIndex[i1]], 529 // db.Files[db.FolderStartFileIndex[i2]]); 530 } 531 */ 532 533 /* 534 we sort empty files and dirs in such order: 535 - Dir.NonAnti (name sorted) 536 - File.NonAnti (name sorted) 537 - File.Anti (name sorted) 538 - Dir.Anti (reverse name sorted) 539 */ 540 541 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param) 542 { 543 const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param; 544 const CUpdateItem &u1 = updateItems[*p1]; 545 const CUpdateItem &u2 = updateItems[*p2]; 546 // NonAnti < Anti 547 if (u1.IsAnti != u2.IsAnti) 548 return (u1.IsAnti ? 1 : -1); 549 if (u1.IsDir != u2.IsDir) 550 { 551 // Dir.NonAnti < File < Dir.Anti 552 if (u1.IsDir) 553 return (u1.IsAnti ? 1 : -1); 554 return (u2.IsAnti ? -1 : 1); 555 } 556 int n = CompareFileNames(u1.Name, u2.Name); 557 return (u1.IsDir && u1.IsAnti) ? -n : n; 558 } 559 560 static const char *g_Exts = 561 " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo" 562 " zip jar ear war msi" 563 " 3gp avi mov mpeg mpg mpe wmv" 564 " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav" 565 " swf" 566 " chm hxi hxs" 567 " gif jpeg jpg jp2 png tiff bmp ico psd psp" 568 " awg ps eps cgm dxf svg vrml wmf emf ai md" 569 " cad dwg pps key sxi" 570 " max 3ds" 571 " iso bin nrg mdf img pdi tar cpio xpi" 572 " vfd vhd vud vmc vsv" 573 " vmdk dsk nvram vmem vmsd vmsn vmss vmtm" 574 " inl inc idl acf asa" 575 " h hpp hxx c cpp cxx m mm go swift" 576 " rc java cs rs pas bas vb cls ctl frm dlg def" 577 " f77 f f90 f95" 578 " asm s" 579 " sql manifest dep" 580 " mak clw csproj vcproj sln dsp dsw" 581 " class" 582 " bat cmd bash sh" 583 " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml" 584 " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs" 585 " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf" 586 " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf" 587 " abw afp cwk lwp wpd wps wpt wrf wri" 588 " abf afm bdf fon mgf otf pcf pfa snf ttf" 589 " dbf mdb nsf ntf wdb db fdb gdb" 590 " exe dll ocx vbx sfx sys tlb awx com obj lib out o so" 591 " pdb pch idb ncb opt"; 592 593 static unsigned GetExtIndex(const char *ext) 594 { 595 unsigned extIndex = 1; 596 const char *p = g_Exts; 597 for (;;) 598 { 599 char c = *p++; 600 if (c == 0) 601 return extIndex; 602 if (c == ' ') 603 continue; 604 unsigned pos = 0; 605 for (;;) 606 { 607 char c2 = ext[pos++]; 608 if (c2 == 0 && (c == 0 || c == ' ')) 609 return extIndex; 610 if (c != c2) 611 break; 612 c = *p++; 613 } 614 extIndex++; 615 for (;;) 616 { 617 if (c == 0) 618 return extIndex; 619 if (c == ' ') 620 break; 621 c = *p++; 622 } 623 } 624 } 625 626 struct CRefItem 627 { 628 const CUpdateItem *UpdateItem; 629 UInt32 Index; 630 unsigned ExtensionPos; 631 unsigned NamePos; 632 unsigned ExtensionIndex; 633 634 CRefItem() {}; 635 CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType): 636 UpdateItem(&ui), 637 Index(index), 638 ExtensionPos(0), 639 NamePos(0), 640 ExtensionIndex(0) 641 { 642 if (sortByType) 643 { 644 int slashPos = ui.Name.ReverseFind_PathSepar(); 645 NamePos = slashPos + 1; 646 int dotPos = ui.Name.ReverseFind_Dot(); 647 if (dotPos <= slashPos) 648 ExtensionPos = ui.Name.Len(); 649 else 650 { 651 ExtensionPos = dotPos + 1; 652 if (ExtensionPos != ui.Name.Len()) 653 { 654 AString s; 655 for (unsigned pos = ExtensionPos;; pos++) 656 { 657 wchar_t c = ui.Name[pos]; 658 if (c >= 0x80) 659 break; 660 if (c == 0) 661 { 662 ExtensionIndex = GetExtIndex(s); 663 break; 664 } 665 s += (char)MyCharLower_Ascii((char)c); 666 } 667 } 668 } 669 } 670 } 671 }; 672 673 struct CSortParam 674 { 675 // const CObjectVector<CTreeFolder> *TreeFolders; 676 bool SortByType; 677 }; 678 679 /* 680 we sort files in such order: 681 - Dir.NonAnti (name sorted) 682 - alt streams 683 - Dirs 684 - Dir.Anti (reverse name sorted) 685 */ 686 687 688 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param) 689 { 690 const CRefItem &a1 = *p1; 691 const CRefItem &a2 = *p2; 692 const CUpdateItem &u1 = *a1.UpdateItem; 693 const CUpdateItem &u2 = *a2.UpdateItem; 694 695 /* 696 if (u1.IsAltStream != u2.IsAltStream) 697 return u1.IsAltStream ? 1 : -1; 698 */ 699 700 // Actually there are no dirs that time. They were stored in other steps 701 // So that code is unused? 702 if (u1.IsDir != u2.IsDir) 703 return u1.IsDir ? 1 : -1; 704 if (u1.IsDir) 705 { 706 if (u1.IsAnti != u2.IsAnti) 707 return (u1.IsAnti ? 1 : -1); 708 int n = CompareFileNames(u1.Name, u2.Name); 709 return -n; 710 } 711 712 // bool sortByType = *(bool *)param; 713 const CSortParam *sortParam = (const CSortParam *)param; 714 bool sortByType = sortParam->SortByType; 715 if (sortByType) 716 { 717 RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex); 718 RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos))); 719 RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos))); 720 if (!u1.MTimeDefined && u2.MTimeDefined) return 1; 721 if (u1.MTimeDefined && !u2.MTimeDefined) return -1; 722 if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime); 723 RINOZ_COMP(u1.Size, u2.Size); 724 } 725 /* 726 int par1 = a1.UpdateItem->ParentFolderIndex; 727 int par2 = a2.UpdateItem->ParentFolderIndex; 728 const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1]; 729 const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2]; 730 731 int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd; 732 int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd; 733 if (b1 < b2) 734 { 735 if (e1 <= b2) 736 return -1; 737 // p2 in p1 738 int par = par2; 739 for (;;) 740 { 741 const CTreeFolder &tf = (*sortParam->TreeFolders)[par]; 742 par = tf.Parent; 743 if (par == par1) 744 { 745 RINOZ(CompareFileNames(u1.Name, tf.Name)); 746 break; 747 } 748 } 749 } 750 else if (b2 < b1) 751 { 752 if (e2 <= b1) 753 return 1; 754 // p1 in p2 755 int par = par1; 756 for (;;) 757 { 758 const CTreeFolder &tf = (*sortParam->TreeFolders)[par]; 759 par = tf.Parent; 760 if (par == par2) 761 { 762 RINOZ(CompareFileNames(tf.Name, u2.Name)); 763 break; 764 } 765 } 766 } 767 */ 768 // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex); 769 RINOK(CompareFileNames(u1.Name, u2.Name)); 770 RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient); 771 RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive); 772 return 0; 773 } 774 775 struct CSolidGroup 776 { 777 CRecordVector<UInt32> Indices; 778 779 CRecordVector<CFolderRepack> folderRefs; 780 }; 781 782 static const char * const g_ExeExts[] = 783 { 784 "dll" 785 , "exe" 786 , "ocx" 787 , "sfx" 788 , "sys" 789 }; 790 791 static bool IsExeExt(const wchar_t *ext) 792 { 793 for (unsigned i = 0; i < ARRAY_SIZE(g_ExeExts); i++) 794 if (StringsAreEqualNoCase_Ascii(ext, g_ExeExts[i])) 795 return true; 796 return false; 797 } 798 799 struct CAnalysis 800 { 801 CMyComPtr<IArchiveUpdateCallbackFile> Callback; 802 CByteBuffer Buffer; 803 804 bool ParseWav; 805 bool ParseExe; 806 bool ParseAll; 807 808 CAnalysis(): 809 ParseWav(true), 810 ParseExe(false), 811 ParseAll(false) 812 {} 813 814 HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode); 815 }; 816 817 static const size_t kAnalysisBufSize = 1 << 14; 818 819 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode) 820 { 821 filterMode.Id = 0; 822 filterMode.Delta = 0; 823 824 CFilterMode filterModeTemp = filterMode; 825 826 int slashPos = ui.Name.ReverseFind_PathSepar(); 827 int dotPos = ui.Name.ReverseFind_Dot(); 828 829 // if (dotPos > slashPos) 830 { 831 bool needReadFile = ParseAll; 832 833 bool probablyIsSameIsa = false; 834 835 if (!needReadFile || !Callback) 836 { 837 const wchar_t *ext; 838 if (dotPos > slashPos) 839 ext = ui.Name.Ptr(dotPos + 1); 840 else 841 ext = ui.Name.RightPtr(0); 842 843 // p7zip uses the trick to store posix attributes in high 16 bits 844 if (ui.Attrib & 0x8000) 845 { 846 unsigned st_mode = ui.Attrib >> 16; 847 // st_mode = 00111; 848 if ((st_mode & 00111) && (ui.Size >= 2048)) 849 { 850 #ifndef _WIN32 851 probablyIsSameIsa = true; 852 #endif 853 needReadFile = true; 854 } 855 } 856 857 if (IsExeExt(ext)) 858 { 859 needReadFile = true; 860 #ifdef _WIN32 861 probablyIsSameIsa = true; 862 needReadFile = ParseExe; 863 #endif 864 } 865 else if (StringsAreEqualNoCase_Ascii(ext, "wav")) 866 { 867 needReadFile = ParseWav; 868 } 869 /* 870 else if (!needReadFile && ParseUnixExt) 871 { 872 if (StringsAreEqualNoCase_Ascii(ext, "so") 873 || StringsAreEqualNoCase_Ascii(ext, "")) 874 875 needReadFile = true; 876 } 877 */ 878 } 879 880 if (needReadFile && Callback) 881 { 882 if (Buffer.Size() != kAnalysisBufSize) 883 { 884 Buffer.Alloc(kAnalysisBufSize); 885 } 886 { 887 CMyComPtr<ISequentialInStream> stream; 888 HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze); 889 if (result == S_OK && stream) 890 { 891 size_t size = kAnalysisBufSize; 892 result = ReadStream(stream, Buffer, &size); 893 stream.Release(); 894 // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK)); 895 if (result == S_OK) 896 { 897 BoolInt parseRes = ParseFile(Buffer, size, &filterModeTemp); 898 if (parseRes && filterModeTemp.Delta == 0) 899 { 900 filterModeTemp.SetDelta(); 901 if (filterModeTemp.Delta != 0 && filterModeTemp.Id != k_Delta) 902 { 903 if (ui.Size % filterModeTemp.Delta != 0) 904 { 905 parseRes = false; 906 } 907 } 908 } 909 if (!parseRes) 910 { 911 filterModeTemp.Id = 0; 912 filterModeTemp.Delta = 0; 913 } 914 } 915 } 916 } 917 } 918 else if ((needReadFile && !Callback) || probablyIsSameIsa) 919 { 920 #ifdef MY_CPU_X86_OR_AMD64 921 if (probablyIsSameIsa) 922 filterModeTemp.Id = k_X86; 923 #endif 924 } 925 } 926 927 filterMode = filterModeTemp; 928 return S_OK; 929 } 930 931 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m) 932 { 933 m.Id = methodID; 934 m.NumStreams = numStreams; 935 } 936 937 static HRESULT AddBondForFilter(CCompressionMethodMode &mode) 938 { 939 for (unsigned c = 1; c < mode.Methods.Size(); c++) 940 { 941 if (!mode.IsThereBond_to_Coder(c)) 942 { 943 CBond2 bond; 944 bond.OutCoder = 0; 945 bond.OutStream = 0; 946 bond.InCoder = c; 947 mode.Bonds.Add(bond); 948 return S_OK; 949 } 950 } 951 return E_INVALIDARG; 952 } 953 954 static HRESULT AddFilterBond(CCompressionMethodMode &mode) 955 { 956 if (!mode.Bonds.IsEmpty()) 957 return AddBondForFilter(mode); 958 return S_OK; 959 } 960 961 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode) 962 { 963 // mode.Methods[0] must be k_BCJ2 method ! 964 965 CMethodFull m; 966 GetMethodFull(k_LZMA, 1, m); 967 968 m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20); 969 m.AddProp32(NCoderPropID::kNumFastBytes, 128); 970 m.AddProp32(NCoderPropID::kNumThreads, 1); 971 m.AddProp32(NCoderPropID::kLitPosBits, 2); 972 m.AddProp32(NCoderPropID::kLitContextBits, 0); 973 // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2"); 974 975 unsigned methodIndex = mode.Methods.Size(); 976 977 if (mode.Bonds.IsEmpty()) 978 { 979 for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++) 980 { 981 CBond2 bond; 982 bond.OutCoder = i; 983 bond.OutStream = 0; 984 bond.InCoder = i + 1; 985 mode.Bonds.Add(bond); 986 } 987 } 988 989 mode.Methods.Add(m); 990 mode.Methods.Add(m); 991 992 RINOK(AddBondForFilter(mode)); 993 CBond2 bond; 994 bond.OutCoder = 0; 995 bond.InCoder = methodIndex; bond.OutStream = 1; mode.Bonds.Add(bond); 996 bond.InCoder = methodIndex + 1; bond.OutStream = 2; mode.Bonds.Add(bond); 997 return S_OK; 998 } 999 1000 static HRESULT MakeExeMethod(CCompressionMethodMode &mode, 1001 const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter) 1002 { 1003 if (mode.Filter_was_Inserted) 1004 { 1005 const CMethodFull &m = mode.Methods[0]; 1006 CMethodId id = m.Id; 1007 if (id == k_BCJ2) 1008 return AddBcj2Methods(mode); 1009 if (!m.IsSimpleCoder()) 1010 return E_NOTIMPL; 1011 // if (Bonds.IsEmpty()) we can create bonds later 1012 return AddFilterBond(mode); 1013 } 1014 1015 if (filterMode.Id == 0) 1016 return S_OK; 1017 1018 CMethodFull &m = mode.Methods.InsertNew(0); 1019 1020 { 1021 FOR_VECTOR(k, mode.Bonds) 1022 { 1023 CBond2 &bond = mode.Bonds[k]; 1024 bond.InCoder++; 1025 bond.OutCoder++; 1026 } 1027 } 1028 1029 HRESULT res; 1030 1031 if (bcj2Filter && Is86Filter(filterMode.Id)) 1032 { 1033 GetMethodFull(k_BCJ2, 4, m); 1034 res = AddBcj2Methods(mode); 1035 } 1036 else 1037 { 1038 GetMethodFull(filterMode.Id, 1, m); 1039 if (filterMode.Id == k_Delta) 1040 m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta); 1041 res = AddFilterBond(mode); 1042 1043 int alignBits = -1; 1044 if (filterMode.Id == k_Delta || filterMode.Delta != 0) 1045 { 1046 if (filterMode.Delta == 1) alignBits = 0; 1047 else if (filterMode.Delta == 2) alignBits = 1; 1048 else if (filterMode.Delta == 4) alignBits = 2; 1049 else if (filterMode.Delta == 8) alignBits = 3; 1050 else if (filterMode.Delta == 16) alignBits = 4; 1051 } 1052 else 1053 { 1054 // alignBits = GetAlignForFilterMethod(filterMode.Id); 1055 } 1056 1057 if (res == S_OK && alignBits >= 0) 1058 { 1059 unsigned nextCoder = 1; 1060 if (!mode.Bonds.IsEmpty()) 1061 { 1062 nextCoder = mode.Bonds.Back().InCoder; 1063 } 1064 if (nextCoder < mode.Methods.Size()) 1065 { 1066 CMethodFull &nextMethod = mode.Methods[nextCoder]; 1067 if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2) 1068 { 1069 if (!nextMethod.Are_Lzma_Model_Props_Defined()) 1070 { 1071 if (alignBits != 0) 1072 { 1073 if (alignBits > 2 || filterMode.Id == k_Delta) 1074 nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits); 1075 unsigned lc = 0; 1076 if (alignBits < 3) 1077 lc = 3 - alignBits; 1078 nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc); 1079 nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits); 1080 } 1081 } 1082 } 1083 } 1084 } 1085 } 1086 1087 return res; 1088 } 1089 1090 1091 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2) 1092 { 1093 file2.Attrib = ui.Attrib; file2.AttribDefined = ui.AttribDefined; 1094 file2.CTime = ui.CTime; file2.CTimeDefined = ui.CTimeDefined; 1095 file2.ATime = ui.ATime; file2.ATimeDefined = ui.ATimeDefined; 1096 file2.MTime = ui.MTime; file2.MTimeDefined = ui.MTimeDefined; 1097 file2.IsAnti = ui.IsAnti; 1098 // file2.IsAux = false; 1099 file2.StartPosDefined = false; 1100 // file2.StartPos = 0; 1101 } 1102 1103 1104 static void UpdateItem_To_FileItem(const CUpdateItem &ui, 1105 CFileItem &file, CFileItem2 &file2) 1106 { 1107 UpdateItem_To_FileItem2(ui, file2); 1108 1109 file.Size = ui.Size; 1110 file.IsDir = ui.IsDir; 1111 file.HasStream = ui.HasStream(); 1112 // file.IsAltStream = ui.IsAltStream; 1113 } 1114 1115 1116 1117 class CRepackInStreamWithSizes: 1118 public ISequentialInStream, 1119 public ICompressGetSubStreamSize, 1120 public CMyUnknownImp 1121 { 1122 CMyComPtr<ISequentialInStream> _stream; 1123 // UInt64 _size; 1124 const CBoolVector *_extractStatuses; 1125 UInt32 _startIndex; 1126 public: 1127 const CDbEx *_db; 1128 1129 void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses) 1130 { 1131 _startIndex = startIndex; 1132 _extractStatuses = extractStatuses; 1133 // _size = 0; 1134 _stream = stream; 1135 } 1136 // UInt64 GetSize() const { return _size; } 1137 1138 MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize) 1139 1140 STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); 1141 1142 STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value); 1143 }; 1144 1145 STDMETHODIMP CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize) 1146 { 1147 return _stream->Read(data, size, processedSize); 1148 /* 1149 UInt32 realProcessedSize; 1150 HRESULT result = _stream->Read(data, size, &realProcessedSize); 1151 _size += realProcessedSize; 1152 if (processedSize) 1153 *processedSize = realProcessedSize; 1154 return result; 1155 */ 1156 } 1157 1158 STDMETHODIMP CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value) 1159 { 1160 *value = 0; 1161 if (subStream >= _extractStatuses->Size()) 1162 return S_FALSE; // E_FAIL; 1163 unsigned index = (unsigned)subStream; 1164 if ((*_extractStatuses)[index]) 1165 { 1166 const CFileItem &fi = _db->Files[_startIndex + index]; 1167 if (fi.HasStream) 1168 *value = fi.Size; 1169 } 1170 return S_OK; 1171 } 1172 1173 1174 class CRepackStreamBase 1175 { 1176 protected: 1177 bool _needWrite; 1178 bool _fileIsOpen; 1179 bool _calcCrc; 1180 UInt32 _crc; 1181 UInt64 _rem; 1182 1183 const CBoolVector *_extractStatuses; 1184 UInt32 _startIndex; 1185 unsigned _currentIndex; 1186 1187 HRESULT OpenFile(); 1188 HRESULT CloseFile(); 1189 HRESULT ProcessEmptyFiles(); 1190 1191 public: 1192 const CDbEx *_db; 1193 CMyComPtr<IArchiveUpdateCallbackFile> _opCallback; 1194 CMyComPtr<IArchiveExtractCallbackMessage> _extractCallback; 1195 1196 HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses); 1197 HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; } 1198 }; 1199 1200 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses) 1201 { 1202 _startIndex = startIndex; 1203 _extractStatuses = extractStatuses; 1204 1205 _currentIndex = 0; 1206 _fileIsOpen = false; 1207 1208 return ProcessEmptyFiles(); 1209 } 1210 1211 HRESULT CRepackStreamBase::OpenFile() 1212 { 1213 UInt32 arcIndex = _startIndex + _currentIndex; 1214 const CFileItem &fi = _db->Files[arcIndex]; 1215 1216 _needWrite = (*_extractStatuses)[_currentIndex]; 1217 if (_opCallback) 1218 { 1219 RINOK(_opCallback->ReportOperation( 1220 NEventIndexType::kInArcIndex, arcIndex, 1221 _needWrite ? 1222 NUpdateNotifyOp::kRepack : 1223 NUpdateNotifyOp::kSkip)); 1224 } 1225 1226 _crc = CRC_INIT_VAL; 1227 _calcCrc = (fi.CrcDefined && !fi.IsDir); 1228 1229 _fileIsOpen = true; 1230 _rem = fi.Size; 1231 return S_OK; 1232 } 1233 1234 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002; 1235 1236 HRESULT CRepackStreamBase::CloseFile() 1237 { 1238 UInt32 arcIndex = _startIndex + _currentIndex; 1239 const CFileItem &fi = _db->Files[arcIndex]; 1240 _fileIsOpen = false; 1241 _currentIndex++; 1242 if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc)) 1243 return S_OK; 1244 1245 if (_extractCallback) 1246 { 1247 RINOK(_extractCallback->ReportExtractResult( 1248 NEventIndexType::kInArcIndex, arcIndex, 1249 NExtract::NOperationResult::kCRCError)); 1250 } 1251 // return S_FALSE; 1252 return k_My_HRESULT_CRC_ERROR; 1253 } 1254 1255 HRESULT CRepackStreamBase::ProcessEmptyFiles() 1256 { 1257 while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0) 1258 { 1259 RINOK(OpenFile()); 1260 RINOK(CloseFile()); 1261 } 1262 return S_OK; 1263 } 1264 1265 1266 1267 #ifndef _7ZIP_ST 1268 1269 class CFolderOutStream2: 1270 public CRepackStreamBase, 1271 public ISequentialOutStream, 1272 public CMyUnknownImp 1273 { 1274 public: 1275 CMyComPtr<ISequentialOutStream> _stream; 1276 1277 MY_UNKNOWN_IMP 1278 1279 STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); 1280 }; 1281 1282 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize) 1283 { 1284 if (processedSize) 1285 *processedSize = 0; 1286 1287 while (size != 0) 1288 { 1289 if (_fileIsOpen) 1290 { 1291 UInt32 cur = (size < _rem ? size : (UInt32)_rem); 1292 HRESULT result = S_OK; 1293 if (_needWrite) 1294 result = _stream->Write(data, cur, &cur); 1295 if (_calcCrc) 1296 _crc = CrcUpdate(_crc, data, cur); 1297 if (processedSize) 1298 *processedSize += cur; 1299 data = (const Byte *)data + cur; 1300 size -= cur; 1301 _rem -= cur; 1302 if (_rem == 0) 1303 { 1304 RINOK(CloseFile()); 1305 RINOK(ProcessEmptyFiles()); 1306 } 1307 RINOK(result); 1308 if (cur == 0) 1309 break; 1310 continue; 1311 } 1312 1313 RINOK(ProcessEmptyFiles()); 1314 if (_currentIndex == _extractStatuses->Size()) 1315 { 1316 // we don't support write cut here 1317 return E_FAIL; 1318 } 1319 RINOK(OpenFile()); 1320 } 1321 1322 return S_OK; 1323 } 1324 1325 #endif 1326 1327 1328 1329 static const UInt32 kTempBufSize = 1 << 16; 1330 1331 class CFolderInStream2: 1332 public CRepackStreamBase, 1333 public ISequentialInStream, 1334 public CMyUnknownImp 1335 { 1336 Byte *_buf; 1337 public: 1338 CMyComPtr<ISequentialInStream> _inStream; 1339 HRESULT Result; 1340 1341 MY_UNKNOWN_IMP 1342 1343 CFolderInStream2(): 1344 Result(S_OK) 1345 { 1346 _buf = new Byte[kTempBufSize]; 1347 } 1348 1349 ~CFolderInStream2() 1350 { 1351 delete []_buf; 1352 } 1353 1354 void Init() { Result = S_OK; } 1355 STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); 1356 }; 1357 1358 STDMETHODIMP CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize) 1359 { 1360 if (processedSize) 1361 *processedSize = 0; 1362 1363 while (size != 0) 1364 { 1365 if (_fileIsOpen) 1366 { 1367 UInt32 cur = (size < _rem ? size : (UInt32)_rem); 1368 1369 void *buf; 1370 if (_needWrite) 1371 buf = data; 1372 else 1373 { 1374 buf = _buf; 1375 if (cur > kTempBufSize) 1376 cur = kTempBufSize; 1377 } 1378 1379 HRESULT result = _inStream->Read(buf, cur, &cur); 1380 _crc = CrcUpdate(_crc, buf, cur); 1381 _rem -= cur; 1382 1383 if (_needWrite) 1384 { 1385 data = (Byte *)data + cur; 1386 size -= cur; 1387 if (processedSize) 1388 *processedSize += cur; 1389 } 1390 1391 if (result != S_OK) 1392 Result = result; 1393 1394 if (_rem == 0) 1395 { 1396 RINOK(CloseFile()); 1397 RINOK(ProcessEmptyFiles()); 1398 } 1399 1400 RINOK(result); 1401 1402 if (cur == 0) 1403 return E_FAIL; 1404 1405 continue; 1406 } 1407 1408 RINOK(ProcessEmptyFiles()); 1409 if (_currentIndex == _extractStatuses->Size()) 1410 { 1411 return S_OK; 1412 } 1413 RINOK(OpenFile()); 1414 } 1415 1416 return S_OK; 1417 } 1418 1419 1420 class CThreadDecoder 1421 #ifndef _7ZIP_ST 1422 : public CVirtThread 1423 #endif 1424 { 1425 public: 1426 CDecoder Decoder; 1427 1428 CThreadDecoder(bool multiThreadMixer): 1429 Decoder(multiThreadMixer) 1430 { 1431 #ifndef _7ZIP_ST 1432 if (multiThreadMixer) 1433 { 1434 MtMode = false; 1435 NumThreads = 1; 1436 FosSpec = new CFolderOutStream2; 1437 Fos = FosSpec; 1438 Result = E_FAIL; 1439 } 1440 #endif 1441 // UnpackSize = 0; 1442 // send_UnpackSize = false; 1443 } 1444 1445 #ifndef _7ZIP_ST 1446 1447 bool dataAfterEnd_Error; 1448 HRESULT Result; 1449 CMyComPtr<IInStream> InStream; 1450 1451 CFolderOutStream2 *FosSpec; 1452 CMyComPtr<ISequentialOutStream> Fos; 1453 1454 UInt64 StartPos; 1455 const CFolders *Folders; 1456 int FolderIndex; 1457 1458 // bool send_UnpackSize; 1459 // UInt64 UnpackSize; 1460 1461 #ifndef _NO_CRYPTO 1462 CMyComPtr<ICryptoGetTextPassword> getTextPassword; 1463 #endif 1464 1465 DECL_EXTERNAL_CODECS_LOC_VARS2; 1466 1467 #ifndef _7ZIP_ST 1468 bool MtMode; 1469 UInt32 NumThreads; 1470 #endif 1471 1472 1473 ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); } 1474 virtual void Execute(); 1475 1476 #endif 1477 }; 1478 1479 #ifndef _7ZIP_ST 1480 1481 void CThreadDecoder::Execute() 1482 { 1483 try 1484 { 1485 #ifndef _NO_CRYPTO 1486 bool isEncrypted = false; 1487 bool passwordIsDefined = false; 1488 UString password; 1489 #endif 1490 1491 dataAfterEnd_Error = false; 1492 1493 Result = Decoder.Decode( 1494 EXTERNAL_CODECS_LOC_VARS 1495 InStream, 1496 StartPos, 1497 *Folders, FolderIndex, 1498 1499 // send_UnpackSize ? &UnpackSize : NULL, 1500 NULL, // unpackSize : FULL unpack 1501 1502 Fos, 1503 NULL, // compressProgress 1504 1505 NULL // *inStreamMainRes 1506 , dataAfterEnd_Error 1507 1508 _7Z_DECODER_CRYPRO_VARS 1509 #ifndef _7ZIP_ST 1510 , MtMode, NumThreads, 1511 0 // MemUsage 1512 #endif 1513 1514 ); 1515 } 1516 catch(...) 1517 { 1518 Result = E_FAIL; 1519 } 1520 1521 /* 1522 if (Result == S_OK) 1523 Result = FosSpec->CheckFinishedState(); 1524 */ 1525 FosSpec->_stream.Release(); 1526 } 1527 1528 #endif 1529 1530 #ifndef _NO_CRYPTO 1531 1532 class CCryptoGetTextPassword: 1533 public ICryptoGetTextPassword, 1534 public CMyUnknownImp 1535 { 1536 public: 1537 UString Password; 1538 1539 MY_UNKNOWN_IMP 1540 STDMETHOD(CryptoGetTextPassword)(BSTR *password); 1541 }; 1542 1543 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password) 1544 { 1545 return StringToBstr(Password, password); 1546 } 1547 1548 #endif 1549 1550 1551 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2) 1552 { 1553 file = inDb.Files[index]; 1554 file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime); 1555 file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime); 1556 file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime); 1557 file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos); 1558 file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib); 1559 file2.IsAnti = inDb.IsItemAnti(index); 1560 // file2.IsAux = inDb.IsItemAux(index); 1561 } 1562 1563 HRESULT Update( 1564 DECL_EXTERNAL_CODECS_LOC_VARS 1565 IInStream *inStream, 1566 const CDbEx *db, 1567 const CObjectVector<CUpdateItem> &updateItems, 1568 // const CObjectVector<CTreeFolder> &treeFolders, 1569 // const CUniqBlocks &secureBlocks, 1570 COutArchive &archive, 1571 CArchiveDatabaseOut &newDatabase, 1572 ISequentialOutStream *seqOutStream, 1573 IArchiveUpdateCallback *updateCallback, 1574 const CUpdateOptions &options 1575 #ifndef _NO_CRYPTO 1576 , ICryptoGetTextPassword *getDecoderPassword 1577 #endif 1578 ) 1579 { 1580 UInt64 numSolidFiles = options.NumSolidFiles; 1581 if (numSolidFiles == 0) 1582 numSolidFiles = 1; 1583 1584 CMyComPtr<IArchiveUpdateCallbackFile> opCallback; 1585 updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); 1586 1587 CMyComPtr<IArchiveExtractCallbackMessage> extractCallback; 1588 updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback); 1589 1590 // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes(); 1591 1592 /* 1593 CMyComPtr<IOutStream> outStream; 1594 RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream)); 1595 if (!outStream) 1596 return E_NOTIMPL; 1597 */ 1598 1599 UInt64 startBlockSize = db ? db->ArcInfo.StartPosition: 0; 1600 if (startBlockSize > 0 && !options.RemoveSfxBlock) 1601 { 1602 RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL)); 1603 } 1604 1605 CIntArr fileIndexToUpdateIndexMap; 1606 UInt64 complexity = 0; 1607 UInt64 inSizeForReduce2 = 0; 1608 bool needEncryptedRepack = false; 1609 1610 CRecordVector<CFilterMode2> filters; 1611 CObjectVector<CSolidGroup> groups; 1612 bool thereAreRepacks = false; 1613 1614 bool useFilters = options.UseFilters; 1615 if (useFilters) 1616 { 1617 const CCompressionMethodMode &method = *options.Method; 1618 1619 FOR_VECTOR (i, method.Methods) 1620 if (IsFilterMethod(method.Methods[i].Id)) 1621 { 1622 useFilters = false; 1623 break; 1624 } 1625 } 1626 1627 if (db) 1628 { 1629 fileIndexToUpdateIndexMap.Alloc(db->Files.Size()); 1630 unsigned i; 1631 1632 for (i = 0; i < db->Files.Size(); i++) 1633 fileIndexToUpdateIndexMap[i] = -1; 1634 1635 for (i = 0; i < updateItems.Size(); i++) 1636 { 1637 int index = updateItems[i].IndexInArchive; 1638 if (index != -1) 1639 fileIndexToUpdateIndexMap[(unsigned)index] = i; 1640 } 1641 1642 for (i = 0; i < db->NumFolders; i++) 1643 { 1644 CNum indexInFolder = 0; 1645 CNum numCopyItems = 0; 1646 CNum numUnpackStreams = db->NumUnpackStreamsVector[i]; 1647 UInt64 repackSize = 0; 1648 1649 for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++) 1650 { 1651 if (fi >= db->Files.Size()) 1652 return E_FAIL; 1653 1654 const CFileItem &file = db->Files[fi]; 1655 if (file.HasStream) 1656 { 1657 indexInFolder++; 1658 int updateIndex = fileIndexToUpdateIndexMap[fi]; 1659 if (updateIndex >= 0 && !updateItems[updateIndex].NewData) 1660 { 1661 numCopyItems++; 1662 repackSize += file.Size; 1663 } 1664 } 1665 } 1666 1667 if (numCopyItems == 0) 1668 continue; 1669 1670 CFolderRepack rep; 1671 rep.FolderIndex = i; 1672 rep.NumCopyFiles = numCopyItems; 1673 CFolderEx f; 1674 db->ParseFolderEx(i, f); 1675 1676 const bool isEncrypted = f.IsEncrypted(); 1677 const bool needCopy = (numCopyItems == numUnpackStreams); 1678 const bool extractFilter = (useFilters || needCopy); 1679 1680 unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter); 1681 1682 while (groupIndex >= groups.Size()) 1683 groups.AddNew(); 1684 1685 groups[groupIndex].folderRefs.Add(rep); 1686 1687 if (needCopy) 1688 complexity += db->GetFolderFullPackSize(i); 1689 else 1690 { 1691 thereAreRepacks = true; 1692 complexity += repackSize; 1693 if (inSizeForReduce2 < repackSize) 1694 inSizeForReduce2 = repackSize; 1695 if (isEncrypted) 1696 needEncryptedRepack = true; 1697 } 1698 } 1699 } 1700 1701 UInt64 inSizeForReduce = 0; 1702 { 1703 bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0); 1704 FOR_VECTOR (i, updateItems) 1705 { 1706 const CUpdateItem &ui = updateItems[i]; 1707 if (ui.NewData) 1708 { 1709 complexity += ui.Size; 1710 if (isSolid) 1711 inSizeForReduce += ui.Size; 1712 else if (inSizeForReduce < ui.Size) 1713 inSizeForReduce = ui.Size; 1714 } 1715 } 1716 } 1717 1718 if (inSizeForReduce < inSizeForReduce2) 1719 inSizeForReduce = inSizeForReduce2; 1720 1721 RINOK(updateCallback->SetTotal(complexity)); 1722 1723 CLocalProgress *lps = new CLocalProgress; 1724 CMyComPtr<ICompressProgressInfo> progress = lps; 1725 lps->Init(updateCallback, true); 1726 1727 #ifndef _7ZIP_ST 1728 1729 CStreamBinder sb; 1730 if (options.MultiThreadMixer) 1731 { 1732 RINOK(sb.CreateEvents()); 1733 } 1734 1735 #endif 1736 1737 CThreadDecoder threadDecoder(options.MultiThreadMixer); 1738 1739 #ifndef _7ZIP_ST 1740 if (options.MultiThreadMixer && thereAreRepacks) 1741 { 1742 #ifdef EXTERNAL_CODECS 1743 threadDecoder.__externalCodecs = __externalCodecs; 1744 #endif 1745 RINOK(threadDecoder.Create()); 1746 } 1747 #endif 1748 1749 { 1750 CAnalysis analysis; 1751 if (options.AnalysisLevel == 0) 1752 { 1753 analysis.ParseWav = false; 1754 analysis.ParseExe = false; 1755 analysis.ParseAll = false; 1756 } 1757 else 1758 { 1759 analysis.Callback = opCallback; 1760 if (options.AnalysisLevel > 0) 1761 { 1762 analysis.ParseWav = true; 1763 if (options.AnalysisLevel >= 7) 1764 { 1765 analysis.ParseExe = true; 1766 if (options.AnalysisLevel >= 9) 1767 analysis.ParseAll = true; 1768 } 1769 } 1770 } 1771 1772 // ---------- Split files to groups ---------- 1773 1774 const CCompressionMethodMode &method = *options.Method; 1775 1776 FOR_VECTOR (i, updateItems) 1777 { 1778 const CUpdateItem &ui = updateItems[i]; 1779 if (!ui.NewData || !ui.HasStream()) 1780 continue; 1781 1782 CFilterMode2 fm; 1783 if (useFilters) 1784 { 1785 RINOK(analysis.GetFilterGroup(i, ui, fm)); 1786 } 1787 fm.Encrypted = method.PasswordIsDefined; 1788 1789 unsigned groupIndex = GetGroup(filters, fm); 1790 while (groupIndex >= groups.Size()) 1791 groups.AddNew(); 1792 groups[groupIndex].Indices.Add(i); 1793 } 1794 } 1795 1796 1797 #ifndef _NO_CRYPTO 1798 1799 CCryptoGetTextPassword *getPasswordSpec = NULL; 1800 CMyComPtr<ICryptoGetTextPassword> getTextPassword; 1801 if (needEncryptedRepack) 1802 { 1803 getPasswordSpec = new CCryptoGetTextPassword; 1804 getTextPassword = getPasswordSpec; 1805 1806 #ifndef _7ZIP_ST 1807 threadDecoder.getTextPassword = getPasswordSpec; 1808 #endif 1809 1810 if (options.Method->PasswordIsDefined) 1811 getPasswordSpec->Password = options.Method->Password; 1812 else 1813 { 1814 if (!getDecoderPassword) 1815 return E_NOTIMPL; 1816 CMyComBSTR password; 1817 RINOK(getDecoderPassword->CryptoGetTextPassword(&password)); 1818 if (password) 1819 getPasswordSpec->Password = password; 1820 } 1821 } 1822 1823 #endif 1824 1825 1826 // ---------- Compress ---------- 1827 1828 RINOK(archive.Create(seqOutStream, false)); 1829 RINOK(archive.SkipPrefixArchiveHeader()); 1830 1831 /* 1832 CIntVector treeFolderToArcIndex; 1833 treeFolderToArcIndex.Reserve(treeFolders.Size()); 1834 for (i = 0; i < treeFolders.Size(); i++) 1835 treeFolderToArcIndex.Add(-1); 1836 // ---------- Write Tree (only AUX dirs) ---------- 1837 for (i = 1; i < treeFolders.Size(); i++) 1838 { 1839 const CTreeFolder &treeFolder = treeFolders[i]; 1840 CFileItem file; 1841 CFileItem2 file2; 1842 file2.Init(); 1843 int secureID = 0; 1844 if (treeFolder.UpdateItemIndex < 0) 1845 { 1846 // we can store virtual dir item wuthout attrib, but we want all items have attrib. 1847 file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY); 1848 file2.IsAux = true; 1849 } 1850 else 1851 { 1852 const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex]; 1853 // if item is not dir, then it's parent for alt streams. 1854 // we will write such items later 1855 if (!ui.IsDir) 1856 continue; 1857 secureID = ui.SecureIndex; 1858 if (ui.NewProps) 1859 UpdateItem_To_FileItem(ui, file, file2); 1860 else 1861 GetFile(*db, ui.IndexInArchive, file, file2); 1862 } 1863 file.Size = 0; 1864 file.HasStream = false; 1865 file.IsDir = true; 1866 file.Parent = treeFolder.Parent; 1867 1868 treeFolderToArcIndex[i] = newDatabase.Files.Size(); 1869 newDatabase.AddFile(file, file2, treeFolder.Name); 1870 1871 if (totalSecureDataSize != 0) 1872 newDatabase.SecureIDs.Add(secureID); 1873 } 1874 */ 1875 1876 { 1877 /* ---------- Write non-AUX dirs and Empty files ---------- */ 1878 CUIntVector emptyRefs; 1879 1880 unsigned i; 1881 1882 for (i = 0; i < updateItems.Size(); i++) 1883 { 1884 const CUpdateItem &ui = updateItems[i]; 1885 if (ui.NewData) 1886 { 1887 if (ui.HasStream()) 1888 continue; 1889 } 1890 else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream) 1891 continue; 1892 /* 1893 if (ui.TreeFolderIndex >= 0) 1894 continue; 1895 */ 1896 emptyRefs.Add(i); 1897 } 1898 1899 emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems); 1900 1901 for (i = 0; i < emptyRefs.Size(); i++) 1902 { 1903 const CUpdateItem &ui = updateItems[emptyRefs[i]]; 1904 CFileItem file; 1905 CFileItem2 file2; 1906 UString name; 1907 if (ui.NewProps) 1908 { 1909 UpdateItem_To_FileItem(ui, file, file2); 1910 file.CrcDefined = false; 1911 name = ui.Name; 1912 } 1913 else 1914 { 1915 GetFile(*db, ui.IndexInArchive, file, file2); 1916 db->GetPath(ui.IndexInArchive, name); 1917 } 1918 1919 /* 1920 if (totalSecureDataSize != 0) 1921 newDatabase.SecureIDs.Add(ui.SecureIndex); 1922 file.Parent = ui.ParentFolderIndex; 1923 */ 1924 newDatabase.AddFile(file, file2, name); 1925 } 1926 } 1927 1928 lps->ProgressOffset = 0; 1929 1930 { 1931 // ---------- Sort Filters ---------- 1932 1933 FOR_VECTOR (i, filters) 1934 { 1935 filters[i].GroupIndex = i; 1936 } 1937 filters.Sort2(); 1938 } 1939 1940 for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++) 1941 { 1942 const CFilterMode2 &filterMode = filters[groupIndex]; 1943 1944 CCompressionMethodMode method = *options.Method; 1945 { 1946 HRESULT res = MakeExeMethod(method, filterMode, 1947 #ifdef _7ZIP_ST 1948 false 1949 #else 1950 options.MaxFilter && options.MultiThreadMixer 1951 #endif 1952 ); 1953 1954 RINOK(res); 1955 } 1956 1957 if (filterMode.Encrypted) 1958 { 1959 if (!method.PasswordIsDefined) 1960 { 1961 #ifndef _NO_CRYPTO 1962 if (getPasswordSpec) 1963 method.Password = getPasswordSpec->Password; 1964 #endif 1965 method.PasswordIsDefined = true; 1966 } 1967 } 1968 else 1969 { 1970 method.PasswordIsDefined = false; 1971 method.Password.Empty(); 1972 } 1973 1974 CEncoder encoder(method); 1975 1976 // ---------- Repack and copy old solid blocks ---------- 1977 1978 const CSolidGroup &group = groups[filterMode.GroupIndex]; 1979 1980 FOR_VECTOR(folderRefIndex, group.folderRefs) 1981 { 1982 const CFolderRepack &rep = group.folderRefs[folderRefIndex]; 1983 1984 unsigned folderIndex = rep.FolderIndex; 1985 1986 CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex]; 1987 1988 if (rep.NumCopyFiles == numUnpackStreams) 1989 { 1990 if (opCallback) 1991 { 1992 RINOK(opCallback->ReportOperation( 1993 NEventIndexType::kBlockIndex, (UInt32)folderIndex, 1994 NUpdateNotifyOp::kReplicate)); 1995 1996 // ---------- Copy old solid block ---------- 1997 { 1998 CNum indexInFolder = 0; 1999 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++) 2000 { 2001 if (db->Files[fi].HasStream) 2002 { 2003 indexInFolder++; 2004 RINOK(opCallback->ReportOperation( 2005 NEventIndexType::kInArcIndex, (UInt32)fi, 2006 NUpdateNotifyOp::kReplicate)); 2007 } 2008 } 2009 } 2010 } 2011 2012 UInt64 packSize = db->GetFolderFullPackSize(folderIndex); 2013 RINOK(WriteRange(inStream, archive.SeqStream, 2014 db->GetFolderStreamPos(folderIndex, 0), packSize, progress)); 2015 lps->ProgressOffset += packSize; 2016 2017 CFolder &folder = newDatabase.Folders.AddNew(); 2018 db->ParseFolderInfo(folderIndex, folder); 2019 CNum startIndex = db->FoStartPackStreamIndex[folderIndex]; 2020 FOR_VECTOR(j, folder.PackStreams) 2021 { 2022 newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j)); 2023 // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]); 2024 // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]); 2025 } 2026 2027 size_t indexStart = db->FoToCoderUnpackSizes[folderIndex]; 2028 size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1]; 2029 for (; indexStart < indexEnd; indexStart++) 2030 newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]); 2031 } 2032 else 2033 { 2034 // ---------- Repack old solid block ---------- 2035 2036 CBoolVector extractStatuses; 2037 2038 CNum indexInFolder = 0; 2039 2040 if (opCallback) 2041 { 2042 RINOK(opCallback->ReportOperation( 2043 NEventIndexType::kBlockIndex, (UInt32)folderIndex, 2044 NUpdateNotifyOp::kRepack)) 2045 } 2046 2047 /* We could reduce data size of decoded folder, if we don't need to repack 2048 last files in folder. But the gain in speed is small in most cases. 2049 So we unpack full folder. */ 2050 2051 UInt64 sizeToEncode = 0; 2052 2053 /* 2054 UInt64 importantUnpackSize = 0; 2055 unsigned numImportantFiles = 0; 2056 UInt64 decodeSize = 0; 2057 */ 2058 2059 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++) 2060 { 2061 bool needExtract = false; 2062 const CFileItem &file = db->Files[fi]; 2063 2064 if (file.HasStream) 2065 { 2066 indexInFolder++; 2067 int updateIndex = fileIndexToUpdateIndexMap[fi]; 2068 if (updateIndex >= 0 && !updateItems[updateIndex].NewData) 2069 needExtract = true; 2070 // decodeSize += file.Size; 2071 } 2072 2073 extractStatuses.Add(needExtract); 2074 if (needExtract) 2075 { 2076 sizeToEncode += file.Size; 2077 /* 2078 numImportantFiles = extractStatuses.Size(); 2079 importantUnpackSize = decodeSize; 2080 */ 2081 } 2082 } 2083 2084 // extractStatuses.DeleteFrom(numImportantFiles); 2085 2086 unsigned startPackIndex = newDatabase.PackSizes.Size(); 2087 UInt64 curUnpackSize; 2088 { 2089 2090 CMyComPtr<ISequentialInStream> sbInStream; 2091 CRepackStreamBase *repackBase; 2092 CFolderInStream2 *FosSpec2 = NULL; 2093 2094 CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes; 2095 CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec; 2096 { 2097 #ifndef _7ZIP_ST 2098 if (options.MultiThreadMixer) 2099 { 2100 repackBase = threadDecoder.FosSpec; 2101 CMyComPtr<ISequentialOutStream> sbOutStream; 2102 sb.CreateStreams(&sbInStream, &sbOutStream); 2103 sb.ReInit(); 2104 2105 threadDecoder.FosSpec->_stream = sbOutStream; 2106 2107 threadDecoder.InStream = inStream; 2108 threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0); 2109 threadDecoder.Folders = (const CFolders *)db; 2110 threadDecoder.FolderIndex = folderIndex; 2111 2112 // threadDecoder.UnpackSize = importantUnpackSize; 2113 // threadDecoder.send_UnpackSize = true; 2114 } 2115 else 2116 #endif 2117 { 2118 FosSpec2 = new CFolderInStream2; 2119 FosSpec2->Init(); 2120 sbInStream = FosSpec2; 2121 repackBase = FosSpec2; 2122 2123 #ifndef _NO_CRYPTO 2124 bool isEncrypted = false; 2125 bool passwordIsDefined = false; 2126 UString password; 2127 #endif 2128 2129 CMyComPtr<ISequentialInStream> decodedStream; 2130 bool dataAfterEnd_Error = false; 2131 2132 HRESULT res = threadDecoder.Decoder.Decode( 2133 EXTERNAL_CODECS_LOC_VARS 2134 inStream, 2135 db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);, 2136 *db, folderIndex, 2137 // &importantUnpackSize, // *unpackSize 2138 NULL, // *unpackSize : FULL unpack 2139 2140 NULL, // *outStream 2141 NULL, // *compressProgress 2142 2143 &decodedStream 2144 , dataAfterEnd_Error 2145 2146 _7Z_DECODER_CRYPRO_VARS 2147 #ifndef _7ZIP_ST 2148 , false // mtMode 2149 , 1 // numThreads 2150 , 0 // memUsage 2151 #endif 2152 ); 2153 2154 RINOK(res); 2155 if (!decodedStream) 2156 return E_FAIL; 2157 2158 FosSpec2->_inStream = decodedStream; 2159 } 2160 2161 repackBase->_db = db; 2162 repackBase->_opCallback = opCallback; 2163 repackBase->_extractCallback = extractCallback; 2164 2165 UInt32 startIndex = db->FolderStartFileIndex[folderIndex]; 2166 RINOK(repackBase->Init(startIndex, &extractStatuses)); 2167 2168 inStreamSizeCountSpec->_db = db; 2169 inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses); 2170 2171 #ifndef _7ZIP_ST 2172 if (options.MultiThreadMixer) 2173 { 2174 threadDecoder.Start(); 2175 } 2176 #endif 2177 } 2178 2179 curUnpackSize = sizeToEncode; 2180 2181 HRESULT encodeRes = encoder.Encode( 2182 EXTERNAL_CODECS_LOC_VARS 2183 inStreamSizeCount, 2184 // NULL, 2185 &inSizeForReduce, 2186 newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize, 2187 archive.SeqStream, newDatabase.PackSizes, progress); 2188 2189 if (encodeRes == k_My_HRESULT_CRC_ERROR) 2190 return E_FAIL; 2191 2192 #ifndef _7ZIP_ST 2193 if (options.MultiThreadMixer) 2194 { 2195 // 16.00: hang was fixed : for case if decoding was not finished. 2196 // We close CBinderInStream and it calls CStreamBinder::CloseRead() 2197 inStreamSizeCount.Release(); 2198 sbInStream.Release(); 2199 2200 threadDecoder.WaitExecuteFinish(); 2201 2202 HRESULT decodeRes = threadDecoder.Result; 2203 // if (res == k_My_HRESULT_CRC_ERROR) 2204 if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error) 2205 { 2206 if (extractCallback) 2207 { 2208 RINOK(extractCallback->ReportExtractResult( 2209 NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex], 2210 // NEventIndexType::kBlockIndex, (UInt32)folderIndex, 2211 (decodeRes != S_OK ? 2212 NExtract::NOperationResult::kDataError : 2213 NExtract::NOperationResult::kDataAfterEnd))); 2214 } 2215 if (decodeRes != S_OK) 2216 return E_FAIL; 2217 } 2218 RINOK(decodeRes); 2219 if (encodeRes == S_OK) 2220 if (sb.ProcessedSize != sizeToEncode) 2221 encodeRes = E_FAIL; 2222 } 2223 else 2224 #endif 2225 { 2226 if (FosSpec2->Result == S_FALSE) 2227 { 2228 if (extractCallback) 2229 { 2230 RINOK(extractCallback->ReportExtractResult( 2231 NEventIndexType::kBlockIndex, (UInt32)folderIndex, 2232 NExtract::NOperationResult::kDataError)); 2233 } 2234 return E_FAIL; 2235 } 2236 RINOK(FosSpec2->Result); 2237 } 2238 2239 RINOK(encodeRes); 2240 RINOK(repackBase->CheckFinishedState()); 2241 2242 if (curUnpackSize != sizeToEncode) 2243 return E_FAIL; 2244 } 2245 2246 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++) 2247 lps->OutSize += newDatabase.PackSizes[startPackIndex]; 2248 lps->InSize += curUnpackSize; 2249 } 2250 2251 newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles); 2252 2253 CNum indexInFolder = 0; 2254 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++) 2255 { 2256 if (db->Files[fi].HasStream) 2257 { 2258 indexInFolder++; 2259 int updateIndex = fileIndexToUpdateIndexMap[fi]; 2260 if (updateIndex >= 0) 2261 { 2262 const CUpdateItem &ui = updateItems[updateIndex]; 2263 if (ui.NewData) 2264 continue; 2265 2266 UString name; 2267 CFileItem file; 2268 CFileItem2 file2; 2269 GetFile(*db, fi, file, file2); 2270 2271 if (ui.NewProps) 2272 { 2273 UpdateItem_To_FileItem2(ui, file2); 2274 file.IsDir = ui.IsDir; 2275 name = ui.Name; 2276 } 2277 else 2278 db->GetPath(fi, name); 2279 2280 /* 2281 file.Parent = ui.ParentFolderIndex; 2282 if (ui.TreeFolderIndex >= 0) 2283 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size(); 2284 if (totalSecureDataSize != 0) 2285 newDatabase.SecureIDs.Add(ui.SecureIndex); 2286 */ 2287 newDatabase.AddFile(file, file2, name); 2288 } 2289 } 2290 } 2291 } 2292 2293 2294 // ---------- Compress files to new solid blocks ---------- 2295 2296 unsigned numFiles = group.Indices.Size(); 2297 if (numFiles == 0) 2298 continue; 2299 CRecordVector<CRefItem> refItems; 2300 refItems.ClearAndSetSize(numFiles); 2301 // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1 2302 bool sortByType = options.UseTypeSorting; 2303 2304 unsigned i; 2305 2306 for (i = 0; i < numFiles; i++) 2307 refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType); 2308 2309 CSortParam sortParam; 2310 // sortParam.TreeFolders = &treeFolders; 2311 sortParam.SortByType = sortByType; 2312 refItems.Sort(CompareUpdateItems, (void *)&sortParam); 2313 2314 CObjArray<UInt32> indices(numFiles); 2315 2316 for (i = 0; i < numFiles; i++) 2317 { 2318 UInt32 index = refItems[i].Index; 2319 indices[i] = index; 2320 /* 2321 const CUpdateItem &ui = updateItems[index]; 2322 CFileItem file; 2323 if (ui.NewProps) 2324 UpdateItem_To_FileItem(ui, file); 2325 else 2326 file = db.Files[ui.IndexInArchive]; 2327 if (file.IsAnti || file.IsDir) 2328 return E_FAIL; 2329 newDatabase.Files.Add(file); 2330 */ 2331 } 2332 2333 for (i = 0; i < numFiles;) 2334 { 2335 UInt64 totalSize = 0; 2336 unsigned numSubFiles; 2337 2338 const wchar_t *prevExtension = NULL; 2339 2340 for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++) 2341 { 2342 const CUpdateItem &ui = updateItems[indices[i + numSubFiles]]; 2343 totalSize += ui.Size; 2344 if (totalSize > options.NumSolidBytes) 2345 break; 2346 if (options.SolidExtension) 2347 { 2348 int slashPos = ui.Name.ReverseFind_PathSepar(); 2349 int dotPos = ui.Name.ReverseFind_Dot(); 2350 const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1); 2351 if (numSubFiles == 0) 2352 prevExtension = ext; 2353 else if (!StringsAreEqualNoCase(ext, prevExtension)) 2354 break; 2355 } 2356 } 2357 2358 if (numSubFiles < 1) 2359 numSubFiles = 1; 2360 2361 RINOK(lps->SetCur()); 2362 2363 CFolderInStream *inStreamSpec = new CFolderInStream; 2364 CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec); 2365 inStreamSpec->Init(updateCallback, &indices[i], numSubFiles); 2366 2367 unsigned startPackIndex = newDatabase.PackSizes.Size(); 2368 UInt64 curFolderUnpackSize = totalSize; 2369 // curFolderUnpackSize = (UInt64)(Int64)-1; 2370 2371 RINOK(encoder.Encode( 2372 EXTERNAL_CODECS_LOC_VARS 2373 solidInStream, 2374 // NULL, 2375 &inSizeForReduce, 2376 newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize, 2377 archive.SeqStream, newDatabase.PackSizes, progress)); 2378 2379 if (!inStreamSpec->WasFinished()) 2380 return E_FAIL; 2381 2382 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++) 2383 lps->OutSize += newDatabase.PackSizes[startPackIndex]; 2384 2385 lps->InSize += curFolderUnpackSize; 2386 // for () 2387 // newDatabase.PackCRCsDefined.Add(false); 2388 // newDatabase.PackCRCs.Add(0); 2389 2390 CNum numUnpackStreams = 0; 2391 UInt64 skippedSize = 0; 2392 2393 for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++) 2394 { 2395 const CUpdateItem &ui = updateItems[indices[i + subIndex]]; 2396 CFileItem file; 2397 CFileItem2 file2; 2398 UString name; 2399 if (ui.NewProps) 2400 { 2401 UpdateItem_To_FileItem(ui, file, file2); 2402 name = ui.Name; 2403 } 2404 else 2405 { 2406 GetFile(*db, ui.IndexInArchive, file, file2); 2407 db->GetPath(ui.IndexInArchive, name); 2408 } 2409 if (file2.IsAnti || file.IsDir) 2410 return E_FAIL; 2411 2412 /* 2413 CFileItem &file = newDatabase.Files[ 2414 startFileIndexInDatabase + i + subIndex]; 2415 */ 2416 if (!inStreamSpec->Processed[subIndex]) 2417 { 2418 skippedSize += ui.Size; 2419 continue; 2420 // file.Name += ".locked"; 2421 } 2422 2423 file.Crc = inStreamSpec->CRCs[subIndex]; 2424 file.Size = inStreamSpec->Sizes[subIndex]; 2425 2426 // if (file.Size >= 0) // test purposes 2427 if (file.Size != 0) 2428 { 2429 file.CrcDefined = true; 2430 file.HasStream = true; 2431 numUnpackStreams++; 2432 } 2433 else 2434 { 2435 file.CrcDefined = false; 2436 file.HasStream = false; 2437 } 2438 2439 /* 2440 file.Parent = ui.ParentFolderIndex; 2441 if (ui.TreeFolderIndex >= 0) 2442 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size(); 2443 if (totalSecureDataSize != 0) 2444 newDatabase.SecureIDs.Add(ui.SecureIndex); 2445 */ 2446 newDatabase.AddFile(file, file2, name); 2447 } 2448 2449 // numUnpackStreams = 0 is very bad case for locked files 2450 // v3.13 doesn't understand it. 2451 newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams); 2452 i += numSubFiles; 2453 2454 if (skippedSize != 0 && complexity >= skippedSize) 2455 { 2456 complexity -= skippedSize; 2457 RINOK(updateCallback->SetTotal(complexity)); 2458 } 2459 } 2460 } 2461 2462 RINOK(lps->SetCur()); 2463 2464 /* 2465 fileIndexToUpdateIndexMap.ClearAndFree(); 2466 groups.ClearAndFree(); 2467 */ 2468 2469 /* 2470 for (i = 0; i < newDatabase.Files.Size(); i++) 2471 { 2472 CFileItem &file = newDatabase.Files[i]; 2473 file.Parent = treeFolderToArcIndex[file.Parent]; 2474 } 2475 2476 if (totalSecureDataSize != 0) 2477 { 2478 newDatabase.SecureBuf.SetCapacity(totalSecureDataSize); 2479 size_t pos = 0; 2480 newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size()); 2481 for (i = 0; i < secureBlocks.Sorted.Size(); i++) 2482 { 2483 const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]]; 2484 size_t size = buf.GetCapacity(); 2485 if (size != 0) 2486 memcpy(newDatabase.SecureBuf + pos, buf, size); 2487 newDatabase.SecureSizes.Add((UInt32)size); 2488 pos += size; 2489 } 2490 } 2491 */ 2492 newDatabase.ReserveDown(); 2493 2494 if (opCallback) 2495 RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader)); 2496 2497 return S_OK; 2498 } 2499 2500 }} 2501