1 /* 2 * Create a squashfs filesystem. This is a highly compressed read only 3 * filesystem. 4 * 5 * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 6 * 2012, 2013, 2014 7 * Phillip Lougher <phillip (at) squashfs.org.uk> 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 2, 12 * or (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software 21 * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 22 * 23 * mksquashfs.c 24 */ 25 26 #define FALSE 0 27 #define TRUE 1 28 #define MAX_LINE 16384 29 30 #include <pwd.h> 31 #include <grp.h> 32 #include <time.h> 33 #include <unistd.h> 34 #include <stdio.h> 35 #include <stddef.h> 36 #include <sys/types.h> 37 #include <sys/stat.h> 38 #include <fcntl.h> 39 #include <errno.h> 40 #include <dirent.h> 41 #include <string.h> 42 #include <stdlib.h> 43 #include <signal.h> 44 #include <setjmp.h> 45 #include <sys/types.h> 46 #include <sys/mman.h> 47 #include <pthread.h> 48 #include <regex.h> 49 #include <fnmatch.h> 50 #include <sys/wait.h> 51 #include <limits.h> 52 #include <ctype.h> 53 54 #ifndef FNM_EXTMATCH /* glibc extension */ 55 #define FNM_EXTMATCH 0 56 #endif 57 58 #ifndef linux 59 #define __BYTE_ORDER BYTE_ORDER 60 #define __BIG_ENDIAN BIG_ENDIAN 61 #define __LITTLE_ENDIAN LITTLE_ENDIAN 62 #include <sys/sysctl.h> 63 #else 64 #include <endian.h> 65 #include <sys/sysinfo.h> 66 #endif 67 68 #include "squashfs_fs.h" 69 #include "squashfs_swap.h" 70 #include "mksquashfs.h" 71 #include "sort.h" 72 #include "pseudo.h" 73 #include "compressor.h" 74 #include "xattr.h" 75 #include "action.h" 76 #include "error.h" 77 #include "progressbar.h" 78 #include "info.h" 79 #include "caches-queues-lists.h" 80 #include "read_fs.h" 81 #include "restore.h" 82 #include "process_fragments.h" 83 84 /* ANDROID CHANGES START*/ 85 #ifdef ANDROID 86 #include "android.h" 87 #include "private/android_filesystem_config.h" 88 #include "private/canned_fs_config.h" 89 int android_config = FALSE; 90 char *context_file = NULL; 91 char *mount_point = NULL; 92 char *target_out_path = NULL; 93 fs_config_func_t fs_config_func = NULL; 94 int align_4k_blocks = TRUE; 95 FILE *block_map_file = NULL; 96 #endif 97 /* ANDROID CHANGES END */ 98 99 int delete = FALSE; 100 int fd; 101 struct squashfs_super_block sBlk; 102 103 /* filesystem flags for building */ 104 int comp_opts = FALSE; 105 int no_xattrs = XATTR_DEF; 106 int noX = FALSE; 107 int duplicate_checking = TRUE; 108 int noF = FALSE; 109 int no_fragments = FALSE; 110 int always_use_fragments = FALSE; 111 int noI = FALSE; 112 int noD = FALSE; 113 int silent = TRUE; 114 int exportable = TRUE; 115 int sparse_files = TRUE; 116 int old_exclude = TRUE; 117 int use_regex = FALSE; 118 int nopad = FALSE; 119 int exit_on_error = FALSE; 120 121 long long global_uid = -1, global_gid = -1; 122 123 /* superblock attributes */ 124 int block_size = SQUASHFS_FILE_SIZE, block_log; 125 unsigned int id_count = 0; 126 int file_count = 0, sym_count = 0, dev_count = 0, dir_count = 0, fifo_count = 0, 127 sock_count = 0; 128 129 /* write position within data section */ 130 long long bytes = 0, total_bytes = 0; 131 132 /* in memory directory table - possibly compressed */ 133 char *directory_table = NULL; 134 unsigned int directory_bytes = 0, directory_size = 0, total_directory_bytes = 0; 135 136 /* cached directory table */ 137 char *directory_data_cache = NULL; 138 unsigned int directory_cache_bytes = 0, directory_cache_size = 0; 139 140 /* in memory inode table - possibly compressed */ 141 char *inode_table = NULL; 142 unsigned int inode_bytes = 0, inode_size = 0, total_inode_bytes = 0; 143 144 /* cached inode table */ 145 char *data_cache = NULL; 146 unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; 147 148 /* inode lookup table */ 149 squashfs_inode *inode_lookup_table = NULL; 150 151 /* in memory directory data */ 152 #define I_COUNT_SIZE 128 153 #define DIR_ENTRIES 32 154 #define INODE_HASH_SIZE 65536 155 #define INODE_HASH_MASK (INODE_HASH_SIZE - 1) 156 #define INODE_HASH(dev, ino) (ino & INODE_HASH_MASK) 157 158 struct cached_dir_index { 159 struct squashfs_dir_index index; 160 char *name; 161 }; 162 163 struct directory { 164 unsigned int start_block; 165 unsigned int size; 166 unsigned char *buff; 167 unsigned char *p; 168 unsigned int entry_count; 169 unsigned char *entry_count_p; 170 unsigned int i_count; 171 unsigned int i_size; 172 struct cached_dir_index *index; 173 unsigned char *index_count_p; 174 unsigned int inode_number; 175 }; 176 177 struct inode_info *inode_info[INODE_HASH_SIZE]; 178 179 /* hash tables used to do fast duplicate searches in duplicate check */ 180 struct file_info *dupl[65536]; 181 int dup_files = 0; 182 183 /* exclude file handling */ 184 /* list of exclude dirs/files */ 185 struct exclude_info { 186 dev_t st_dev; 187 ino_t st_ino; 188 }; 189 190 #define EXCLUDE_SIZE 8192 191 int exclude = 0; 192 struct exclude_info *exclude_paths = NULL; 193 int old_excluded(char *filename, struct stat *buf); 194 195 struct path_entry { 196 char *name; 197 regex_t *preg; 198 struct pathname *paths; 199 }; 200 201 struct pathname { 202 int names; 203 struct path_entry *name; 204 }; 205 206 struct pathnames { 207 int count; 208 struct pathname *path[0]; 209 }; 210 #define PATHS_ALLOC_SIZE 10 211 212 struct pathnames *paths = NULL; 213 struct pathname *path = NULL; 214 struct pathname *stickypath = NULL; 215 int excluded(char *name, struct pathnames *paths, struct pathnames **new); 216 217 int fragments = 0; 218 219 #define FRAG_SIZE 32768 220 221 struct squashfs_fragment_entry *fragment_table = NULL; 222 int fragments_outstanding = 0; 223 224 int fragments_locked = FALSE; 225 226 /* current inode number for directories and non directories */ 227 unsigned int inode_no = 1; 228 unsigned int root_inode_number = 0; 229 230 /* list of source dirs/files */ 231 int source = 0; 232 char **source_path; 233 234 /* list of root directory entries read from original filesystem */ 235 int old_root_entries = 0; 236 struct old_root_entry_info { 237 char *name; 238 struct inode_info inode; 239 }; 240 struct old_root_entry_info *old_root_entry; 241 242 /* restore orignal filesystem state if appending to existing filesystem is 243 * cancelled */ 244 int appending = FALSE; 245 char *sdata_cache, *sdirectory_data_cache, *sdirectory_compressed; 246 247 long long sbytes, stotal_bytes; 248 249 unsigned int sinode_bytes, scache_bytes, sdirectory_bytes, 250 sdirectory_cache_bytes, sdirectory_compressed_bytes, 251 stotal_inode_bytes, stotal_directory_bytes, 252 sinode_count = 0, sfile_count, ssym_count, sdev_count, 253 sdir_count, sfifo_count, ssock_count, sdup_files; 254 int sfragments; 255 int threads; 256 257 /* flag whether destination file is a block device */ 258 int block_device = FALSE; 259 260 /* flag indicating whether files are sorted using sort list(s) */ 261 int sorted = FALSE; 262 263 /* save destination file name for deleting on error */ 264 char *destination_file = NULL; 265 266 /* recovery file for abnormal exit on appending */ 267 char *recovery_file = NULL; 268 int recover = TRUE; 269 270 struct id *id_hash_table[ID_ENTRIES]; 271 struct id *id_table[SQUASHFS_IDS], *sid_table[SQUASHFS_IDS]; 272 unsigned int uid_count = 0, guid_count = 0; 273 unsigned int sid_count = 0, suid_count = 0, sguid_count = 0; 274 275 struct cache *reader_buffer, *fragment_buffer, *reserve_cache; 276 struct cache *bwriter_buffer, *fwriter_buffer; 277 struct queue *to_reader, *to_deflate, *to_writer, *from_writer, 278 *to_frag, *locked_fragment, *to_process_frag; 279 struct seq_queue *to_main; 280 pthread_t reader_thread, writer_thread, main_thread; 281 pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; 282 pthread_t *restore_thread = NULL; 283 pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER; 284 pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER; 285 pthread_mutex_t dup_mutex = PTHREAD_MUTEX_INITIALIZER; 286 287 /* user options that control parallelisation */ 288 int processors = -1; 289 int bwriter_size; 290 291 /* compression operations */ 292 struct compressor *comp = NULL; 293 int compressor_opt_parsed = FALSE; 294 void *stream = NULL; 295 296 /* xattr stats */ 297 unsigned int xattr_bytes = 0, total_xattr_bytes = 0; 298 299 /* fragment to file mapping used when appending */ 300 int append_fragments = 0; 301 struct append_file **file_mapping; 302 303 /* root of the in-core directory structure */ 304 struct dir_info *root_dir; 305 306 static char *read_from_disk(long long start, unsigned int avail_bytes); 307 void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, 308 int type); 309 struct file_info *duplicate(long long file_size, long long bytes, 310 unsigned int **block_list, long long *start, struct fragment **fragment, 311 struct file_buffer *file_buffer, int blocks, unsigned short checksum, 312 int checksum_flag); 313 struct dir_info *dir_scan1(char *, char *, struct pathnames *, 314 struct dir_ent *(_readdir)(struct dir_info *), int); 315 void dir_scan2(struct dir_info *dir, struct pseudo *pseudo); 316 void dir_scan3(struct dir_info *dir); 317 void dir_scan4(struct dir_info *dir); 318 void dir_scan5(struct dir_info *dir); 319 void dir_scan6(struct dir_info *dir); 320 void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info); 321 struct file_info *add_non_dup(long long file_size, long long bytes, 322 unsigned int *block_list, long long start, struct fragment *fragment, 323 unsigned short checksum, unsigned short fragment_checksum, 324 int checksum_flag, int checksum_frag_flag); 325 long long generic_write_table(int, void *, int, void *, int); 326 void restorefs(); 327 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth); 328 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad); 329 unsigned short get_checksum_mem(char *buff, int bytes); 330 void check_usable_phys_mem(int total_mem); 331 332 333 void prep_exit() 334 { 335 if(restore_thread) { 336 if(pthread_self() == *restore_thread) { 337 /* 338 * Recursive failure when trying to restore filesystem! 339 * Nothing to do except to exit, otherwise we'll just 340 * appear to hang. The user should be able to restore 341 * from the recovery file (which is why it was added, in 342 * case of catastrophic failure in Mksquashfs) 343 */ 344 exit(1); 345 } else { 346 /* signal the restore thread to restore */ 347 pthread_kill(*restore_thread, SIGUSR1); 348 pthread_exit(NULL); 349 } 350 } else if(delete) { 351 if(destination_file && !block_device) 352 unlink(destination_file); 353 } else if(recovery_file) 354 unlink(recovery_file); 355 } 356 357 358 int add_overflow(int a, int b) 359 { 360 return (INT_MAX - a) < b; 361 } 362 363 364 int shift_overflow(int a, int shift) 365 { 366 return (INT_MAX >> shift) < a; 367 } 368 369 370 int multiply_overflow(int a, int multiplier) 371 { 372 return (INT_MAX / multiplier) < a; 373 } 374 375 376 int multiply_overflowll(long long a, int multiplier) 377 { 378 return (LLONG_MAX / multiplier) < a; 379 } 380 381 382 #define MKINODE(A) ((squashfs_inode)(((squashfs_inode) inode_bytes << 16) \ 383 + (((char *)A) - data_cache))) 384 385 386 void restorefs() 387 { 388 ERROR("Exiting - restoring original filesystem!\n\n"); 389 390 bytes = sbytes; 391 memcpy(data_cache, sdata_cache, cache_bytes = scache_bytes); 392 memcpy(directory_data_cache, sdirectory_data_cache, 393 sdirectory_cache_bytes); 394 directory_cache_bytes = sdirectory_cache_bytes; 395 inode_bytes = sinode_bytes; 396 directory_bytes = sdirectory_bytes; 397 memcpy(directory_table + directory_bytes, sdirectory_compressed, 398 sdirectory_compressed_bytes); 399 directory_bytes += sdirectory_compressed_bytes; 400 total_bytes = stotal_bytes; 401 total_inode_bytes = stotal_inode_bytes; 402 total_directory_bytes = stotal_directory_bytes; 403 inode_count = sinode_count; 404 file_count = sfile_count; 405 sym_count = ssym_count; 406 dev_count = sdev_count; 407 dir_count = sdir_count; 408 fifo_count = sfifo_count; 409 sock_count = ssock_count; 410 dup_files = sdup_files; 411 fragments = sfragments; 412 id_count = sid_count; 413 restore_xattrs(); 414 write_filesystem_tables(&sBlk, nopad); 415 exit(1); 416 } 417 418 419 void sighandler() 420 { 421 EXIT_MKSQUASHFS(); 422 } 423 424 425 int mangle2(void *strm, char *d, char *s, int size, 426 int block_size, int uncompressed, int data_block) 427 { 428 int error, c_byte = 0; 429 430 if(!uncompressed) { 431 c_byte = compressor_compress(comp, strm, d, s, size, block_size, 432 &error); 433 if(c_byte == -1) 434 BAD_ERROR("mangle2:: %s compress failed with error " 435 "code %d\n", comp->name, error); 436 } 437 438 if(c_byte == 0 || c_byte >= size) { 439 memcpy(d, s, size); 440 return size | (data_block ? SQUASHFS_COMPRESSED_BIT_BLOCK : 441 SQUASHFS_COMPRESSED_BIT); 442 } 443 444 return c_byte; 445 } 446 447 448 int mangle(char *d, char *s, int size, int block_size, 449 int uncompressed, int data_block) 450 { 451 return mangle2(stream, d, s, size, block_size, uncompressed, 452 data_block); 453 } 454 455 456 void *get_inode(int req_size) 457 { 458 int data_space; 459 unsigned short c_byte; 460 461 while(cache_bytes >= SQUASHFS_METADATA_SIZE) { 462 if((inode_size - inode_bytes) < 463 ((SQUASHFS_METADATA_SIZE << 1)) + 2) { 464 void *it = realloc(inode_table, inode_size + 465 (SQUASHFS_METADATA_SIZE << 1) + 2); 466 if(it == NULL) 467 MEM_ERROR(); 468 inode_table = it; 469 inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 470 } 471 472 c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, 473 data_cache, SQUASHFS_METADATA_SIZE, 474 SQUASHFS_METADATA_SIZE, noI, 0); 475 TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); 476 SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); 477 inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; 478 total_inode_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; 479 memmove(data_cache, data_cache + SQUASHFS_METADATA_SIZE, 480 cache_bytes - SQUASHFS_METADATA_SIZE); 481 cache_bytes -= SQUASHFS_METADATA_SIZE; 482 } 483 484 data_space = (cache_size - cache_bytes); 485 if(data_space < req_size) { 486 int realloc_size = cache_size == 0 ? 487 ((req_size + SQUASHFS_METADATA_SIZE) & 488 ~(SQUASHFS_METADATA_SIZE - 1)) : req_size - 489 data_space; 490 491 void *dc = realloc(data_cache, cache_size + 492 realloc_size); 493 if(dc == NULL) 494 MEM_ERROR(); 495 cache_size += realloc_size; 496 data_cache = dc; 497 } 498 499 cache_bytes += req_size; 500 501 return data_cache + cache_bytes - req_size; 502 } 503 504 505 int read_bytes(int fd, void *buff, int bytes) 506 { 507 int res, count; 508 509 for(count = 0; count < bytes; count += res) { 510 res = read(fd, buff + count, bytes - count); 511 if(res < 1) { 512 if(res == 0) 513 goto bytes_read; 514 else if(errno != EINTR) { 515 ERROR("Read failed because %s\n", 516 strerror(errno)); 517 return -1; 518 } else 519 res = 0; 520 } 521 } 522 523 bytes_read: 524 return count; 525 } 526 527 528 int read_fs_bytes(int fd, long long byte, int bytes, void *buff) 529 { 530 off_t off = byte; 531 int res = 1; 532 533 TRACE("read_fs_bytes: reading from position 0x%llx, bytes %d\n", 534 byte, bytes); 535 536 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 537 pthread_mutex_lock(&pos_mutex); 538 if(lseek(fd, off, SEEK_SET) == -1) { 539 ERROR("read_fs_bytes: Lseek on destination failed because %s, " 540 "offset=0x%llx\n", strerror(errno), off); 541 res = 0; 542 } else if(read_bytes(fd, buff, bytes) < bytes) { 543 ERROR("Read on destination failed\n"); 544 res = 0; 545 } 546 547 pthread_cleanup_pop(1); 548 return res; 549 } 550 551 552 int write_bytes(int fd, void *buff, int bytes) 553 { 554 int res, count; 555 556 for(count = 0; count < bytes; count += res) { 557 res = write(fd, buff + count, bytes - count); 558 if(res == -1) { 559 if(errno != EINTR) { 560 ERROR("Write failed because %s\n", 561 strerror(errno)); 562 return -1; 563 } 564 res = 0; 565 } 566 } 567 568 return 0; 569 } 570 571 572 void write_destination(int fd, long long byte, int bytes, void *buff) 573 { 574 off_t off = byte; 575 576 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 577 pthread_mutex_lock(&pos_mutex); 578 579 if(lseek(fd, off, SEEK_SET) == -1) { 580 ERROR("write_destination: Lseek on destination " 581 "failed because %s, offset=0x%llx\n", strerror(errno), 582 off); 583 BAD_ERROR("Probably out of space on output %s\n", 584 block_device ? "block device" : "filesystem"); 585 } 586 587 if(write_bytes(fd, buff, bytes) == -1) 588 BAD_ERROR("Failed to write to output %s\n", 589 block_device ? "block device" : "filesystem"); 590 591 pthread_cleanup_pop(1); 592 } 593 594 595 long long write_inodes() 596 { 597 unsigned short c_byte; 598 int avail_bytes; 599 char *datap = data_cache; 600 long long start_bytes = bytes; 601 602 while(cache_bytes) { 603 if(inode_size - inode_bytes < 604 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 605 void *it = realloc(inode_table, inode_size + 606 ((SQUASHFS_METADATA_SIZE << 1) + 2)); 607 if(it == NULL) 608 MEM_ERROR(); 609 inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 610 inode_table = it; 611 } 612 avail_bytes = cache_bytes > SQUASHFS_METADATA_SIZE ? 613 SQUASHFS_METADATA_SIZE : cache_bytes; 614 c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, datap, 615 avail_bytes, SQUASHFS_METADATA_SIZE, noI, 0); 616 TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); 617 SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); 618 inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; 619 total_inode_bytes += avail_bytes + BLOCK_OFFSET; 620 datap += avail_bytes; 621 cache_bytes -= avail_bytes; 622 } 623 624 write_destination(fd, bytes, inode_bytes, inode_table); 625 bytes += inode_bytes; 626 627 return start_bytes; 628 } 629 630 631 long long write_directories() 632 { 633 unsigned short c_byte; 634 int avail_bytes; 635 char *directoryp = directory_data_cache; 636 long long start_bytes = bytes; 637 638 while(directory_cache_bytes) { 639 if(directory_size - directory_bytes < 640 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 641 void *dt = realloc(directory_table, 642 directory_size + ((SQUASHFS_METADATA_SIZE << 1) 643 + 2)); 644 if(dt == NULL) 645 MEM_ERROR(); 646 directory_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 647 directory_table = dt; 648 } 649 avail_bytes = directory_cache_bytes > SQUASHFS_METADATA_SIZE ? 650 SQUASHFS_METADATA_SIZE : directory_cache_bytes; 651 c_byte = mangle(directory_table + directory_bytes + 652 BLOCK_OFFSET, directoryp, avail_bytes, 653 SQUASHFS_METADATA_SIZE, noI, 0); 654 TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, 655 c_byte); 656 SQUASHFS_SWAP_SHORTS(&c_byte, 657 directory_table + directory_bytes, 1); 658 directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + 659 BLOCK_OFFSET; 660 total_directory_bytes += avail_bytes + BLOCK_OFFSET; 661 directoryp += avail_bytes; 662 directory_cache_bytes -= avail_bytes; 663 } 664 write_destination(fd, bytes, directory_bytes, directory_table); 665 bytes += directory_bytes; 666 667 return start_bytes; 668 } 669 670 671 long long write_id_table() 672 { 673 unsigned int id_bytes = SQUASHFS_ID_BYTES(id_count); 674 unsigned int p[id_count]; 675 int i; 676 677 TRACE("write_id_table: ids %d, id_bytes %d\n", id_count, id_bytes); 678 for(i = 0; i < id_count; i++) { 679 TRACE("write_id_table: id index %d, id %d", i, id_table[i]->id); 680 SQUASHFS_SWAP_INTS(&id_table[i]->id, p + i, 1); 681 } 682 683 return generic_write_table(id_bytes, p, 0, NULL, noI); 684 } 685 686 687 struct id *get_id(unsigned int id) 688 { 689 int hash = ID_HASH(id); 690 struct id *entry = id_hash_table[hash]; 691 692 for(; entry; entry = entry->next) 693 if(entry->id == id) 694 break; 695 696 return entry; 697 } 698 699 700 struct id *create_id(unsigned int id) 701 { 702 int hash = ID_HASH(id); 703 struct id *entry = malloc(sizeof(struct id)); 704 if(entry == NULL) 705 MEM_ERROR(); 706 entry->id = id; 707 entry->index = id_count ++; 708 entry->flags = 0; 709 entry->next = id_hash_table[hash]; 710 id_hash_table[hash] = entry; 711 id_table[entry->index] = entry; 712 return entry; 713 } 714 715 716 unsigned int get_uid(unsigned int uid) 717 { 718 struct id *entry = get_id(uid); 719 720 if(entry == NULL) { 721 if(id_count == SQUASHFS_IDS) 722 BAD_ERROR("Out of uids!\n"); 723 entry = create_id(uid); 724 } 725 726 if((entry->flags & ISA_UID) == 0) { 727 entry->flags |= ISA_UID; 728 uid_count ++; 729 } 730 731 return entry->index; 732 } 733 734 735 unsigned int get_guid(unsigned int guid) 736 { 737 struct id *entry = get_id(guid); 738 739 if(entry == NULL) { 740 if(id_count == SQUASHFS_IDS) 741 BAD_ERROR("Out of gids!\n"); 742 entry = create_id(guid); 743 } 744 745 if((entry->flags & ISA_GID) == 0) { 746 entry->flags |= ISA_GID; 747 guid_count ++; 748 } 749 750 return entry->index; 751 } 752 753 754 #define ALLOC_SIZE 128 755 756 char *_pathname(struct dir_ent *dir_ent, char *pathname, int *size) 757 { 758 if(pathname == NULL) { 759 pathname = malloc(ALLOC_SIZE); 760 if(pathname == NULL) 761 MEM_ERROR(); 762 } 763 764 for(;;) { 765 int res = snprintf(pathname, *size, "%s/%s", 766 dir_ent->our_dir->pathname, 767 dir_ent->source_name ? : dir_ent->name); 768 769 if(res < 0) 770 BAD_ERROR("snprintf failed in pathname\n"); 771 else if(res >= *size) { 772 /* 773 * pathname is too small to contain the result, so 774 * increase it and try again 775 */ 776 *size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); 777 pathname = realloc(pathname, *size); 778 if(pathname == NULL) 779 MEM_ERROR(); 780 } else 781 break; 782 } 783 784 return pathname; 785 } 786 787 788 char *pathname(struct dir_ent *dir_ent) 789 { 790 static char *pathname = NULL; 791 static int size = ALLOC_SIZE; 792 793 if (dir_ent->nonstandard_pathname) 794 return dir_ent->nonstandard_pathname; 795 796 return pathname = _pathname(dir_ent, pathname, &size); 797 } 798 799 800 char *pathname_reader(struct dir_ent *dir_ent) 801 { 802 static char *pathname = NULL; 803 static int size = ALLOC_SIZE; 804 805 if (dir_ent->nonstandard_pathname) 806 return dir_ent->nonstandard_pathname; 807 808 return pathname = _pathname(dir_ent, pathname, &size); 809 } 810 811 812 char *subpathname(struct dir_ent *dir_ent) 813 { 814 static char *subpath = NULL; 815 static int size = ALLOC_SIZE; 816 int res; 817 818 if(subpath == NULL) { 819 subpath = malloc(ALLOC_SIZE); 820 if(subpath == NULL) 821 MEM_ERROR(); 822 } 823 824 for(;;) { 825 if(dir_ent->our_dir->subpath[0] != '\0') 826 res = snprintf(subpath, size, "%s/%s", 827 dir_ent->our_dir->subpath, dir_ent->name); 828 else 829 res = snprintf(subpath, size, "/%s", dir_ent->name); 830 831 if(res < 0) 832 BAD_ERROR("snprintf failed in subpathname\n"); 833 else if(res >= size) { 834 /* 835 * subpath is too small to contain the result, so 836 * increase it and try again 837 */ 838 size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); 839 subpath = realloc(subpath, size); 840 if(subpath == NULL) 841 MEM_ERROR(); 842 } else 843 break; 844 } 845 846 return subpath; 847 } 848 849 850 static inline unsigned int get_inode_no(struct inode_info *inode) 851 { 852 return inode->inode_number; 853 } 854 855 856 static inline unsigned int get_parent_no(struct dir_info *dir) 857 { 858 return dir->depth ? get_inode_no(dir->dir_ent->inode) : inode_no; 859 } 860 861 862 /* ANDROID CHANGES START*/ 863 #ifdef ANDROID 864 static inline void write_block_map_entry(char *sub_path, unsigned long long start_block, unsigned long long total_size, 865 char * mount_point, FILE *block_map_file) { 866 if (block_map_file) { 867 unsigned long long round_start = (start_block + (1 << 12) - 1) >> 12; 868 unsigned long long round_end = ((start_block + total_size) >> 12) - 1; 869 if (round_start && total_size && round_start <= round_end) { 870 fprintf(block_map_file, "/%s", mount_point); 871 if (sub_path[0] != '/') fprintf(block_map_file, "/"); 872 if (round_start == round_end) 873 fprintf(block_map_file, "%s %lld\n", sub_path, round_start); 874 else 875 fprintf(block_map_file, "%s %lld-%lld\n", sub_path, round_start, round_end); 876 } 877 } 878 } 879 #endif 880 /* ANDROID CHANGES END */ 881 882 int create_inode(squashfs_inode *i_no, struct dir_info *dir_info, 883 struct dir_ent *dir_ent, int type, long long byte_size, 884 long long start_block, unsigned int offset, unsigned int *block_list, 885 struct fragment *fragment, struct directory *dir_in, long long sparse) 886 { 887 struct stat *buf = &dir_ent->inode->buf; 888 union squashfs_inode_header inode_header; 889 struct squashfs_base_inode_header *base = &inode_header.base; 890 void *inode; 891 char *filename = pathname(dir_ent); 892 int nlink = dir_ent->inode->nlink; 893 int xattr = read_xattrs(dir_ent); 894 895 switch(type) { 896 case SQUASHFS_FILE_TYPE: 897 if(dir_ent->inode->nlink > 1 || 898 byte_size >= (1LL << 32) || 899 start_block >= (1LL << 32) || 900 sparse || IS_XATTR(xattr)) 901 type = SQUASHFS_LREG_TYPE; 902 break; 903 case SQUASHFS_DIR_TYPE: 904 if(dir_info->dir_is_ldir || IS_XATTR(xattr)) 905 type = SQUASHFS_LDIR_TYPE; 906 break; 907 case SQUASHFS_SYMLINK_TYPE: 908 if(IS_XATTR(xattr)) 909 type = SQUASHFS_LSYMLINK_TYPE; 910 break; 911 case SQUASHFS_BLKDEV_TYPE: 912 if(IS_XATTR(xattr)) 913 type = SQUASHFS_LBLKDEV_TYPE; 914 break; 915 case SQUASHFS_CHRDEV_TYPE: 916 if(IS_XATTR(xattr)) 917 type = SQUASHFS_LCHRDEV_TYPE; 918 break; 919 case SQUASHFS_FIFO_TYPE: 920 if(IS_XATTR(xattr)) 921 type = SQUASHFS_LFIFO_TYPE; 922 break; 923 case SQUASHFS_SOCKET_TYPE: 924 if(IS_XATTR(xattr)) 925 type = SQUASHFS_LSOCKET_TYPE; 926 break; 927 } 928 929 base->mode = SQUASHFS_MODE(buf->st_mode); 930 base->uid = get_uid((unsigned int) global_uid == -1 ? 931 buf->st_uid : global_uid); 932 base->inode_type = type; 933 base->guid = get_guid((unsigned int) global_gid == -1 ? 934 buf->st_gid : global_gid); 935 base->mtime = buf->st_mtime; 936 base->inode_number = get_inode_no(dir_ent->inode); 937 938 if(type == SQUASHFS_FILE_TYPE) { 939 int i; 940 struct squashfs_reg_inode_header *reg = &inode_header.reg; 941 size_t off = offsetof(struct squashfs_reg_inode_header, block_list); 942 /* ANDROID CHANGES START*/ 943 #ifdef ANDROID 944 unsigned long long total_size = 0; 945 char *sub_path; 946 #endif 947 /* ANDROID CHANGES END */ 948 949 inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); 950 reg->file_size = byte_size; 951 reg->start_block = start_block; 952 reg->fragment = fragment->index; 953 reg->offset = fragment->offset; 954 SQUASHFS_SWAP_REG_INODE_HEADER(reg, inode); 955 SQUASHFS_SWAP_INTS(block_list, inode + off, offset); 956 TRACE("File inode, file_size %lld, start_block 0x%llx, blocks " 957 "%d, fragment %d, offset %d, size %d\n", byte_size, 958 start_block, offset, fragment->index, fragment->offset, 959 fragment->size); 960 for(i = 0; i < offset; i++) { 961 TRACE("Block %d, size %d\n", i, block_list[i]); 962 total_size += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]); 963 } 964 /* ANDROID CHANGES START*/ 965 #ifdef ANDROID 966 sub_path = subpathname(dir_ent); 967 if (block_map_file && fragment->index == -1) { 968 write_block_map_entry(sub_path, start_block, total_size, mount_point, block_map_file); 969 } 970 #endif 971 /* ANDROID CHANGES END */ 972 } 973 else if(type == SQUASHFS_LREG_TYPE) { 974 /* ANDROID CHANGES START*/ 975 #ifdef ANDROID 976 unsigned long long total_size = 0; 977 char *sub_path; 978 #endif 979 /* ANDROID CHANGES END */ 980 int i; 981 struct squashfs_lreg_inode_header *reg = &inode_header.lreg; 982 size_t off = offsetof(struct squashfs_lreg_inode_header, block_list); 983 984 inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); 985 reg->nlink = nlink; 986 reg->file_size = byte_size; 987 reg->start_block = start_block; 988 reg->fragment = fragment->index; 989 reg->offset = fragment->offset; 990 if(sparse && sparse >= byte_size) 991 sparse = byte_size - 1; 992 reg->sparse = sparse; 993 reg->xattr = xattr; 994 SQUASHFS_SWAP_LREG_INODE_HEADER(reg, inode); 995 SQUASHFS_SWAP_INTS(block_list, inode + off, offset); 996 TRACE("Long file inode, file_size %lld, start_block 0x%llx, " 997 "blocks %d, fragment %d, offset %d, size %d, nlink %d" 998 "\n", byte_size, start_block, offset, fragment->index, 999 fragment->offset, fragment->size, nlink); 1000 for(i = 0; i < offset; i++) { 1001 TRACE("Block %d, size %d\n", i, block_list[i]); 1002 total_size += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]); 1003 } 1004 /* ANDROID CHANGES START*/ 1005 #ifdef ANDROID 1006 sub_path = subpathname(dir_ent); 1007 if (block_map_file && fragment->index == -1) { 1008 write_block_map_entry(sub_path, start_block, total_size, mount_point, block_map_file); 1009 } 1010 #endif 1011 /* ANDROID CHANGES END */ 1012 } 1013 else if(type == SQUASHFS_LDIR_TYPE) { 1014 int i; 1015 unsigned char *p; 1016 struct squashfs_ldir_inode_header *dir = &inode_header.ldir; 1017 struct cached_dir_index *index = dir_in->index; 1018 unsigned int i_count = dir_in->i_count; 1019 unsigned int i_size = dir_in->i_size; 1020 1021 if(byte_size >= 1 << 27) 1022 BAD_ERROR("directory greater than 2^27-1 bytes!\n"); 1023 1024 inode = get_inode(sizeof(*dir) + i_size); 1025 dir->inode_type = SQUASHFS_LDIR_TYPE; 1026 dir->nlink = dir_ent->dir->directory_count + 2; 1027 dir->file_size = byte_size; 1028 dir->offset = offset; 1029 dir->start_block = start_block; 1030 dir->i_count = i_count; 1031 dir->parent_inode = get_parent_no(dir_ent->our_dir); 1032 dir->xattr = xattr; 1033 1034 SQUASHFS_SWAP_LDIR_INODE_HEADER(dir, inode); 1035 p = inode + offsetof(struct squashfs_ldir_inode_header, index); 1036 for(i = 0; i < i_count; i++) { 1037 SQUASHFS_SWAP_DIR_INDEX(&index[i].index, p); 1038 p += offsetof(struct squashfs_dir_index, name); 1039 memcpy(p, index[i].name, index[i].index.size + 1); 1040 p += index[i].index.size + 1; 1041 } 1042 TRACE("Long directory inode, file_size %lld, start_block " 1043 "0x%llx, offset 0x%x, nlink %d\n", byte_size, 1044 start_block, offset, dir_ent->dir->directory_count + 2); 1045 } 1046 else if(type == SQUASHFS_DIR_TYPE) { 1047 struct squashfs_dir_inode_header *dir = &inode_header.dir; 1048 1049 inode = get_inode(sizeof(*dir)); 1050 dir->nlink = dir_ent->dir->directory_count + 2; 1051 dir->file_size = byte_size; 1052 dir->offset = offset; 1053 dir->start_block = start_block; 1054 dir->parent_inode = get_parent_no(dir_ent->our_dir); 1055 SQUASHFS_SWAP_DIR_INODE_HEADER(dir, inode); 1056 TRACE("Directory inode, file_size %lld, start_block 0x%llx, " 1057 "offset 0x%x, nlink %d\n", byte_size, start_block, 1058 offset, dir_ent->dir->directory_count + 2); 1059 } 1060 else if(type == SQUASHFS_CHRDEV_TYPE || type == SQUASHFS_BLKDEV_TYPE) { 1061 struct squashfs_dev_inode_header *dev = &inode_header.dev; 1062 unsigned int major = major(buf->st_rdev); 1063 unsigned int minor = minor(buf->st_rdev); 1064 1065 if(major > 0xfff) { 1066 ERROR("Major %d out of range in device node %s, " 1067 "truncating to %d\n", major, filename, 1068 major & 0xfff); 1069 major &= 0xfff; 1070 } 1071 if(minor > 0xfffff) { 1072 ERROR("Minor %d out of range in device node %s, " 1073 "truncating to %d\n", minor, filename, 1074 minor & 0xfffff); 1075 minor &= 0xfffff; 1076 } 1077 inode = get_inode(sizeof(*dev)); 1078 dev->nlink = nlink; 1079 dev->rdev = (major << 8) | (minor & 0xff) | 1080 ((minor & ~0xff) << 12); 1081 SQUASHFS_SWAP_DEV_INODE_HEADER(dev, inode); 1082 TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); 1083 } 1084 else if(type == SQUASHFS_LCHRDEV_TYPE || type == SQUASHFS_LBLKDEV_TYPE) { 1085 struct squashfs_ldev_inode_header *dev = &inode_header.ldev; 1086 unsigned int major = major(buf->st_rdev); 1087 unsigned int minor = minor(buf->st_rdev); 1088 1089 if(major > 0xfff) { 1090 ERROR("Major %d out of range in device node %s, " 1091 "truncating to %d\n", major, filename, 1092 major & 0xfff); 1093 major &= 0xfff; 1094 } 1095 if(minor > 0xfffff) { 1096 ERROR("Minor %d out of range in device node %s, " 1097 "truncating to %d\n", minor, filename, 1098 minor & 0xfffff); 1099 minor &= 0xfffff; 1100 } 1101 inode = get_inode(sizeof(*dev)); 1102 dev->nlink = nlink; 1103 dev->rdev = (major << 8) | (minor & 0xff) | 1104 ((minor & ~0xff) << 12); 1105 dev->xattr = xattr; 1106 SQUASHFS_SWAP_LDEV_INODE_HEADER(dev, inode); 1107 TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); 1108 } 1109 else if(type == SQUASHFS_SYMLINK_TYPE) { 1110 struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; 1111 int byte = strlen(dir_ent->inode->symlink); 1112 size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); 1113 1114 inode = get_inode(sizeof(*symlink) + byte); 1115 symlink->nlink = nlink; 1116 symlink->symlink_size = byte; 1117 SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); 1118 strncpy(inode + off, dir_ent->inode->symlink, byte); 1119 TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, 1120 nlink); 1121 } 1122 else if(type == SQUASHFS_LSYMLINK_TYPE) { 1123 struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; 1124 int byte = strlen(dir_ent->inode->symlink); 1125 size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); 1126 1127 inode = get_inode(sizeof(*symlink) + byte + 1128 sizeof(unsigned int)); 1129 symlink->nlink = nlink; 1130 symlink->symlink_size = byte; 1131 SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); 1132 strncpy(inode + off, dir_ent->inode->symlink, byte); 1133 SQUASHFS_SWAP_INTS(&xattr, inode + off + byte, 1); 1134 TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, 1135 nlink); 1136 } 1137 else if(type == SQUASHFS_FIFO_TYPE || type == SQUASHFS_SOCKET_TYPE) { 1138 struct squashfs_ipc_inode_header *ipc = &inode_header.ipc; 1139 1140 inode = get_inode(sizeof(*ipc)); 1141 ipc->nlink = nlink; 1142 SQUASHFS_SWAP_IPC_INODE_HEADER(ipc, inode); 1143 TRACE("ipc inode, type %s, nlink %d\n", type == 1144 SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); 1145 } 1146 else if(type == SQUASHFS_LFIFO_TYPE || type == SQUASHFS_LSOCKET_TYPE) { 1147 struct squashfs_lipc_inode_header *ipc = &inode_header.lipc; 1148 1149 inode = get_inode(sizeof(*ipc)); 1150 ipc->nlink = nlink; 1151 ipc->xattr = xattr; 1152 SQUASHFS_SWAP_LIPC_INODE_HEADER(ipc, inode); 1153 TRACE("ipc inode, type %s, nlink %d\n", type == 1154 SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); 1155 } else 1156 BAD_ERROR("Unrecognised inode %d in create_inode\n", type); 1157 1158 *i_no = MKINODE(inode); 1159 inode_count ++; 1160 1161 TRACE("Created inode 0x%llx, type %d, uid %d, guid %d\n", *i_no, type, 1162 base->uid, base->guid); 1163 1164 return TRUE; 1165 } 1166 1167 1168 void add_dir(squashfs_inode inode, unsigned int inode_number, char *name, 1169 int type, struct directory *dir) 1170 { 1171 unsigned char *buff; 1172 struct squashfs_dir_entry idir; 1173 unsigned int start_block = inode >> 16; 1174 unsigned int offset = inode & 0xffff; 1175 unsigned int size = strlen(name); 1176 size_t name_off = offsetof(struct squashfs_dir_entry, name); 1177 1178 if(size > SQUASHFS_NAME_LEN) { 1179 size = SQUASHFS_NAME_LEN; 1180 ERROR("Filename is greater than %d characters, truncating! ..." 1181 "\n", SQUASHFS_NAME_LEN); 1182 } 1183 1184 if(dir->p + sizeof(struct squashfs_dir_entry) + size + 1185 sizeof(struct squashfs_dir_header) 1186 >= dir->buff + dir->size) { 1187 buff = realloc(dir->buff, dir->size += SQUASHFS_METADATA_SIZE); 1188 if(buff == NULL) 1189 MEM_ERROR(); 1190 1191 dir->p = (dir->p - dir->buff) + buff; 1192 if(dir->entry_count_p) 1193 dir->entry_count_p = (dir->entry_count_p - dir->buff + 1194 buff); 1195 dir->index_count_p = dir->index_count_p - dir->buff + buff; 1196 dir->buff = buff; 1197 } 1198 1199 if(dir->entry_count == 256 || start_block != dir->start_block || 1200 ((dir->entry_count_p != NULL) && 1201 ((dir->p + sizeof(struct squashfs_dir_entry) + size - 1202 dir->index_count_p) > SQUASHFS_METADATA_SIZE)) || 1203 ((long long) inode_number - dir->inode_number) > 32767 1204 || ((long long) inode_number - dir->inode_number) 1205 < -32768) { 1206 if(dir->entry_count_p) { 1207 struct squashfs_dir_header dir_header; 1208 1209 if((dir->p + sizeof(struct squashfs_dir_entry) + size - 1210 dir->index_count_p) > 1211 SQUASHFS_METADATA_SIZE) { 1212 if(dir->i_count % I_COUNT_SIZE == 0) { 1213 dir->index = realloc(dir->index, 1214 (dir->i_count + I_COUNT_SIZE) * 1215 sizeof(struct cached_dir_index)); 1216 if(dir->index == NULL) 1217 MEM_ERROR(); 1218 } 1219 dir->index[dir->i_count].index.index = 1220 dir->p - dir->buff; 1221 dir->index[dir->i_count].index.size = size - 1; 1222 dir->index[dir->i_count++].name = name; 1223 dir->i_size += sizeof(struct squashfs_dir_index) 1224 + size; 1225 dir->index_count_p = dir->p; 1226 } 1227 1228 dir_header.count = dir->entry_count - 1; 1229 dir_header.start_block = dir->start_block; 1230 dir_header.inode_number = dir->inode_number; 1231 SQUASHFS_SWAP_DIR_HEADER(&dir_header, 1232 dir->entry_count_p); 1233 1234 } 1235 1236 1237 dir->entry_count_p = dir->p; 1238 dir->start_block = start_block; 1239 dir->entry_count = 0; 1240 dir->inode_number = inode_number; 1241 dir->p += sizeof(struct squashfs_dir_header); 1242 } 1243 1244 idir.offset = offset; 1245 idir.type = type; 1246 idir.size = size - 1; 1247 idir.inode_number = ((long long) inode_number - dir->inode_number); 1248 SQUASHFS_SWAP_DIR_ENTRY(&idir, dir->p); 1249 strncpy((char *) dir->p + name_off, name, size); 1250 dir->p += sizeof(struct squashfs_dir_entry) + size; 1251 dir->entry_count ++; 1252 } 1253 1254 1255 void write_dir(squashfs_inode *inode, struct dir_info *dir_info, 1256 struct directory *dir) 1257 { 1258 unsigned int dir_size = dir->p - dir->buff; 1259 int data_space = directory_cache_size - directory_cache_bytes; 1260 unsigned int directory_block, directory_offset, i_count, index; 1261 unsigned short c_byte; 1262 1263 if(data_space < dir_size) { 1264 int realloc_size = directory_cache_size == 0 ? 1265 ((dir_size + SQUASHFS_METADATA_SIZE) & 1266 ~(SQUASHFS_METADATA_SIZE - 1)) : dir_size - data_space; 1267 1268 void *dc = realloc(directory_data_cache, 1269 directory_cache_size + realloc_size); 1270 if(dc == NULL) 1271 MEM_ERROR(); 1272 directory_cache_size += realloc_size; 1273 directory_data_cache = dc; 1274 } 1275 1276 if(dir_size) { 1277 struct squashfs_dir_header dir_header; 1278 1279 dir_header.count = dir->entry_count - 1; 1280 dir_header.start_block = dir->start_block; 1281 dir_header.inode_number = dir->inode_number; 1282 SQUASHFS_SWAP_DIR_HEADER(&dir_header, dir->entry_count_p); 1283 memcpy(directory_data_cache + directory_cache_bytes, dir->buff, 1284 dir_size); 1285 } 1286 directory_offset = directory_cache_bytes; 1287 directory_block = directory_bytes; 1288 directory_cache_bytes += dir_size; 1289 i_count = 0; 1290 index = SQUASHFS_METADATA_SIZE - directory_offset; 1291 1292 while(1) { 1293 while(i_count < dir->i_count && 1294 dir->index[i_count].index.index < index) 1295 dir->index[i_count++].index.start_block = 1296 directory_bytes; 1297 index += SQUASHFS_METADATA_SIZE; 1298 1299 if(directory_cache_bytes < SQUASHFS_METADATA_SIZE) 1300 break; 1301 1302 if((directory_size - directory_bytes) < 1303 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 1304 void *dt = realloc(directory_table, 1305 directory_size + (SQUASHFS_METADATA_SIZE << 1) 1306 + 2); 1307 if(dt == NULL) 1308 MEM_ERROR(); 1309 directory_size += SQUASHFS_METADATA_SIZE << 1; 1310 directory_table = dt; 1311 } 1312 1313 c_byte = mangle(directory_table + directory_bytes + 1314 BLOCK_OFFSET, directory_data_cache, 1315 SQUASHFS_METADATA_SIZE, SQUASHFS_METADATA_SIZE, 1316 noI, 0); 1317 TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, 1318 c_byte); 1319 SQUASHFS_SWAP_SHORTS(&c_byte, 1320 directory_table + directory_bytes, 1); 1321 directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + 1322 BLOCK_OFFSET; 1323 total_directory_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; 1324 memmove(directory_data_cache, directory_data_cache + 1325 SQUASHFS_METADATA_SIZE, directory_cache_bytes - 1326 SQUASHFS_METADATA_SIZE); 1327 directory_cache_bytes -= SQUASHFS_METADATA_SIZE; 1328 } 1329 1330 create_inode(inode, dir_info, dir_info->dir_ent, SQUASHFS_DIR_TYPE, 1331 dir_size + 3, directory_block, directory_offset, NULL, NULL, 1332 dir, 0); 1333 1334 #ifdef SQUASHFS_TRACE 1335 { 1336 unsigned char *dirp; 1337 int count; 1338 1339 TRACE("Directory contents of inode 0x%llx\n", *inode); 1340 dirp = dir->buff; 1341 while(dirp < dir->p) { 1342 char buffer[SQUASHFS_NAME_LEN + 1]; 1343 struct squashfs_dir_entry idir, *idirp; 1344 struct squashfs_dir_header dirh; 1345 SQUASHFS_SWAP_DIR_HEADER((struct squashfs_dir_header *) dirp, 1346 &dirh); 1347 count = dirh.count + 1; 1348 dirp += sizeof(struct squashfs_dir_header); 1349 1350 TRACE("\tStart block 0x%x, count %d\n", 1351 dirh.start_block, count); 1352 1353 while(count--) { 1354 idirp = (struct squashfs_dir_entry *) dirp; 1355 SQUASHFS_SWAP_DIR_ENTRY(idirp, &idir); 1356 strncpy(buffer, idirp->name, idir.size + 1); 1357 buffer[idir.size + 1] = '\0'; 1358 TRACE("\t\tname %s, inode offset 0x%x, type " 1359 "%d\n", buffer, idir.offset, idir.type); 1360 dirp += sizeof(struct squashfs_dir_entry) + idir.size + 1361 1; 1362 } 1363 } 1364 } 1365 #endif 1366 dir_count ++; 1367 } 1368 1369 1370 static struct file_buffer *get_fragment(struct fragment *fragment) 1371 { 1372 struct squashfs_fragment_entry *disk_fragment; 1373 struct file_buffer *buffer, *compressed_buffer; 1374 long long start_block; 1375 int res, size, index = fragment->index; 1376 char locked; 1377 1378 /* 1379 * Lookup fragment block in cache. 1380 * If the fragment block doesn't exist, then get the compressed version 1381 * from the writer cache or off disk, and decompress it. 1382 * 1383 * This routine has two things which complicate the code: 1384 * 1385 * 1. Multiple threads can simultaneously lookup/create the 1386 * same buffer. This means a buffer needs to be "locked" 1387 * when it is being filled in, to prevent other threads from 1388 * using it when it is not ready. This is because we now do 1389 * fragment duplicate checking in parallel. 1390 * 2. We have two caches which need to be checked for the 1391 * presence of fragment blocks: the normal fragment cache 1392 * and a "reserve" cache. The reserve cache is used to 1393 * prevent an unnecessary pipeline stall when the fragment cache 1394 * is full of fragments waiting to be compressed. 1395 */ 1396 1397 if(fragment->index == SQUASHFS_INVALID_FRAG) 1398 return NULL; 1399 1400 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1401 pthread_mutex_lock(&dup_mutex); 1402 1403 again: 1404 buffer = cache_lookup_nowait(fragment_buffer, index, &locked); 1405 if(buffer) { 1406 pthread_mutex_unlock(&dup_mutex); 1407 if(locked) 1408 /* got a buffer being filled in. Wait for it */ 1409 cache_wait_unlock(buffer); 1410 goto finished; 1411 } 1412 1413 /* not in fragment cache, is it in the reserve cache? */ 1414 buffer = cache_lookup_nowait(reserve_cache, index, &locked); 1415 if(buffer) { 1416 pthread_mutex_unlock(&dup_mutex); 1417 if(locked) 1418 /* got a buffer being filled in. Wait for it */ 1419 cache_wait_unlock(buffer); 1420 goto finished; 1421 } 1422 1423 /* in neither cache, try to get it from the fragment cache */ 1424 buffer = cache_get_nowait(fragment_buffer, index); 1425 if(!buffer) { 1426 /* 1427 * no room, get it from the reserve cache, this is 1428 * dimensioned so it will always have space (no more than 1429 * processors + 1 can have an outstanding reserve buffer) 1430 */ 1431 buffer = cache_get_nowait(reserve_cache, index); 1432 if(!buffer) { 1433 /* failsafe */ 1434 ERROR("no space in reserve cache\n"); 1435 goto again; 1436 } 1437 } 1438 1439 pthread_mutex_unlock(&dup_mutex); 1440 1441 compressed_buffer = cache_lookup(fwriter_buffer, index); 1442 1443 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1444 pthread_mutex_lock(&fragment_mutex); 1445 disk_fragment = &fragment_table[index]; 1446 size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size); 1447 start_block = disk_fragment->start_block; 1448 pthread_cleanup_pop(1); 1449 1450 if(SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) { 1451 int error; 1452 char *data; 1453 1454 if(compressed_buffer) 1455 data = compressed_buffer->data; 1456 else { 1457 data = read_from_disk(start_block, size); 1458 if(data == NULL) { 1459 ERROR("Failed to read fragment from output" 1460 " filesystem\n"); 1461 BAD_ERROR("Output filesystem corrupted?\n"); 1462 } 1463 } 1464 1465 res = compressor_uncompress(comp, buffer->data, data, size, 1466 block_size, &error); 1467 if(res == -1) 1468 BAD_ERROR("%s uncompress failed with error code %d\n", 1469 comp->name, error); 1470 } else if(compressed_buffer) 1471 memcpy(buffer->data, compressed_buffer->data, size); 1472 else { 1473 res = read_fs_bytes(fd, start_block, size, buffer->data); 1474 if(res == 0) { 1475 ERROR("Failed to read fragment from output " 1476 "filesystem\n"); 1477 BAD_ERROR("Output filesystem corrupted?\n"); 1478 } 1479 } 1480 1481 cache_unlock(buffer); 1482 cache_block_put(compressed_buffer); 1483 1484 finished: 1485 pthread_cleanup_pop(0); 1486 1487 return buffer; 1488 } 1489 1490 1491 unsigned short get_fragment_checksum(struct file_info *file) 1492 { 1493 struct file_buffer *frag_buffer; 1494 struct append_file *append; 1495 int res, index = file->fragment->index; 1496 unsigned short checksum; 1497 1498 if(index == SQUASHFS_INVALID_FRAG) 1499 return 0; 1500 1501 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1502 pthread_mutex_lock(&dup_mutex); 1503 res = file->have_frag_checksum; 1504 checksum = file->fragment_checksum; 1505 pthread_cleanup_pop(1); 1506 1507 if(res) 1508 return checksum; 1509 1510 frag_buffer = get_fragment(file->fragment); 1511 1512 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1513 1514 for(append = file_mapping[index]; append; append = append->next) { 1515 int offset = append->file->fragment->offset; 1516 int size = append->file->fragment->size; 1517 unsigned short cksum = 1518 get_checksum_mem(frag_buffer->data + offset, size); 1519 1520 if(file == append->file) 1521 checksum = cksum; 1522 1523 pthread_mutex_lock(&dup_mutex); 1524 append->file->fragment_checksum = cksum; 1525 append->file->have_frag_checksum = TRUE; 1526 pthread_mutex_unlock(&dup_mutex); 1527 } 1528 1529 cache_block_put(frag_buffer); 1530 pthread_cleanup_pop(0); 1531 1532 return checksum; 1533 } 1534 1535 1536 void lock_fragments() 1537 { 1538 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1539 pthread_mutex_lock(&fragment_mutex); 1540 fragments_locked = TRUE; 1541 pthread_cleanup_pop(1); 1542 } 1543 1544 1545 void unlock_fragments() 1546 { 1547 int frg, size; 1548 struct file_buffer *write_buffer; 1549 1550 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1551 pthread_mutex_lock(&fragment_mutex); 1552 1553 /* 1554 * Note queue_empty() is inherently racy with respect to concurrent 1555 * queue get and pushes. We avoid this because we're holding the 1556 * fragment_mutex which ensures no other threads can be using the 1557 * queue at this time. 1558 */ 1559 while(!queue_empty(locked_fragment)) { 1560 write_buffer = queue_get(locked_fragment); 1561 frg = write_buffer->block; 1562 size = SQUASHFS_COMPRESSED_SIZE_BLOCK(fragment_table[frg].size); 1563 fragment_table[frg].start_block = bytes; 1564 write_buffer->block = bytes; 1565 bytes += size; 1566 fragments_outstanding --; 1567 queue_put(to_writer, write_buffer); 1568 TRACE("fragment_locked writing fragment %d, compressed size %d" 1569 "\n", frg, size); 1570 } 1571 fragments_locked = FALSE; 1572 pthread_cleanup_pop(1); 1573 } 1574 1575 /* Called with the fragment_mutex locked */ 1576 void add_pending_fragment(struct file_buffer *write_buffer, int c_byte, 1577 int fragment) 1578 { 1579 fragment_table[fragment].size = c_byte; 1580 write_buffer->block = fragment; 1581 1582 queue_put(locked_fragment, write_buffer); 1583 } 1584 1585 1586 void write_fragment(struct file_buffer *fragment) 1587 { 1588 if(fragment == NULL) 1589 return; 1590 1591 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1592 pthread_mutex_lock(&fragment_mutex); 1593 fragment_table[fragment->block].unused = 0; 1594 fragments_outstanding ++; 1595 queue_put(to_frag, fragment); 1596 pthread_cleanup_pop(1); 1597 } 1598 1599 1600 struct file_buffer *allocate_fragment() 1601 { 1602 struct file_buffer *fragment = cache_get(fragment_buffer, fragments); 1603 1604 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1605 pthread_mutex_lock(&fragment_mutex); 1606 1607 if(fragments % FRAG_SIZE == 0) { 1608 void *ft = realloc(fragment_table, (fragments + 1609 FRAG_SIZE) * sizeof(struct squashfs_fragment_entry)); 1610 if(ft == NULL) 1611 MEM_ERROR(); 1612 fragment_table = ft; 1613 } 1614 1615 fragment->size = 0; 1616 fragment->block = fragments ++; 1617 1618 pthread_cleanup_pop(1); 1619 1620 return fragment; 1621 } 1622 1623 1624 static struct fragment empty_fragment = {SQUASHFS_INVALID_FRAG, 0, 0}; 1625 1626 1627 void free_fragment(struct fragment *fragment) 1628 { 1629 if(fragment != &empty_fragment) 1630 free(fragment); 1631 } 1632 1633 1634 struct fragment *get_and_fill_fragment(struct file_buffer *file_buffer, 1635 struct dir_ent *dir_ent) 1636 { 1637 struct fragment *ffrg; 1638 struct file_buffer **fragment; 1639 1640 if(file_buffer == NULL || file_buffer->size == 0) 1641 return &empty_fragment; 1642 1643 fragment = eval_frag_actions(root_dir, dir_ent); 1644 1645 if((*fragment) && (*fragment)->size + file_buffer->size > block_size) { 1646 write_fragment(*fragment); 1647 *fragment = NULL; 1648 } 1649 1650 ffrg = malloc(sizeof(struct fragment)); 1651 if(ffrg == NULL) 1652 MEM_ERROR(); 1653 1654 if(*fragment == NULL) 1655 *fragment = allocate_fragment(); 1656 1657 ffrg->index = (*fragment)->block; 1658 ffrg->offset = (*fragment)->size; 1659 ffrg->size = file_buffer->size; 1660 memcpy((*fragment)->data + (*fragment)->size, file_buffer->data, 1661 file_buffer->size); 1662 (*fragment)->size += file_buffer->size; 1663 1664 return ffrg; 1665 } 1666 1667 1668 long long generic_write_table(int length, void *buffer, int length2, 1669 void *buffer2, int uncompressed) 1670 { 1671 int meta_blocks = (length + SQUASHFS_METADATA_SIZE - 1) / 1672 SQUASHFS_METADATA_SIZE; 1673 long long *list, start_bytes; 1674 int compressed_size, i, list_size = meta_blocks * sizeof(long long); 1675 unsigned short c_byte; 1676 char cbuffer[(SQUASHFS_METADATA_SIZE << 2) + 2]; 1677 1678 #ifdef SQUASHFS_TRACE 1679 long long obytes = bytes; 1680 int olength = length; 1681 #endif 1682 1683 list = malloc(list_size); 1684 if(list == NULL) 1685 MEM_ERROR(); 1686 1687 for(i = 0; i < meta_blocks; i++) { 1688 int avail_bytes = length > SQUASHFS_METADATA_SIZE ? 1689 SQUASHFS_METADATA_SIZE : length; 1690 c_byte = mangle(cbuffer + BLOCK_OFFSET, buffer + i * 1691 SQUASHFS_METADATA_SIZE , avail_bytes, 1692 SQUASHFS_METADATA_SIZE, uncompressed, 0); 1693 SQUASHFS_SWAP_SHORTS(&c_byte, cbuffer, 1); 1694 list[i] = bytes; 1695 compressed_size = SQUASHFS_COMPRESSED_SIZE(c_byte) + 1696 BLOCK_OFFSET; 1697 TRACE("block %d @ 0x%llx, compressed size %d\n", i, bytes, 1698 compressed_size); 1699 write_destination(fd, bytes, compressed_size, cbuffer); 1700 bytes += compressed_size; 1701 total_bytes += avail_bytes; 1702 length -= avail_bytes; 1703 } 1704 1705 start_bytes = bytes; 1706 if(length2) { 1707 write_destination(fd, bytes, length2, buffer2); 1708 bytes += length2; 1709 total_bytes += length2; 1710 } 1711 1712 SQUASHFS_INSWAP_LONG_LONGS(list, meta_blocks); 1713 write_destination(fd, bytes, list_size, list); 1714 bytes += list_size; 1715 total_bytes += list_size; 1716 1717 TRACE("generic_write_table: total uncompressed %d compressed %lld\n", 1718 olength, bytes - obytes); 1719 1720 free(list); 1721 1722 return start_bytes; 1723 } 1724 1725 1726 long long write_fragment_table() 1727 { 1728 unsigned int frag_bytes = SQUASHFS_FRAGMENT_BYTES(fragments); 1729 int i; 1730 1731 TRACE("write_fragment_table: fragments %d, frag_bytes %d\n", fragments, 1732 frag_bytes); 1733 for(i = 0; i < fragments; i++) { 1734 TRACE("write_fragment_table: fragment %d, start_block 0x%llx, " 1735 "size %d\n", i, fragment_table[i].start_block, 1736 fragment_table[i].size); 1737 SQUASHFS_INSWAP_FRAGMENT_ENTRY(&fragment_table[i]); 1738 } 1739 1740 return generic_write_table(frag_bytes, fragment_table, 0, NULL, noF); 1741 } 1742 1743 1744 char read_from_file_buffer[SQUASHFS_FILE_MAX_SIZE]; 1745 static char *read_from_disk(long long start, unsigned int avail_bytes) 1746 { 1747 int res; 1748 1749 res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer); 1750 if(res == 0) 1751 return NULL; 1752 1753 return read_from_file_buffer; 1754 } 1755 1756 1757 char read_from_file_buffer2[SQUASHFS_FILE_MAX_SIZE]; 1758 char *read_from_disk2(long long start, unsigned int avail_bytes) 1759 { 1760 int res; 1761 1762 res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer2); 1763 if(res == 0) 1764 return NULL; 1765 1766 return read_from_file_buffer2; 1767 } 1768 1769 1770 /* 1771 * Compute 16 bit BSD checksum over the data 1772 */ 1773 unsigned short get_checksum(char *buff, int bytes, unsigned short chksum) 1774 { 1775 unsigned char *b = (unsigned char *) buff; 1776 1777 while(bytes --) { 1778 chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1; 1779 chksum += *b++; 1780 } 1781 1782 return chksum; 1783 } 1784 1785 1786 unsigned short get_checksum_disk(long long start, long long l, 1787 unsigned int *blocks) 1788 { 1789 unsigned short chksum = 0; 1790 unsigned int bytes; 1791 struct file_buffer *write_buffer; 1792 int i; 1793 1794 for(i = 0; l; i++) { 1795 bytes = SQUASHFS_COMPRESSED_SIZE_BLOCK(blocks[i]); 1796 if(bytes == 0) /* sparse block */ 1797 continue; 1798 write_buffer = cache_lookup(bwriter_buffer, start); 1799 if(write_buffer) { 1800 chksum = get_checksum(write_buffer->data, bytes, 1801 chksum); 1802 cache_block_put(write_buffer); 1803 } else { 1804 void *data = read_from_disk(start, bytes); 1805 if(data == NULL) { 1806 ERROR("Failed to checksum data from output" 1807 " filesystem\n"); 1808 BAD_ERROR("Output filesystem corrupted?\n"); 1809 } 1810 1811 chksum = get_checksum(data, bytes, chksum); 1812 } 1813 1814 l -= bytes; 1815 start += bytes; 1816 } 1817 1818 return chksum; 1819 } 1820 1821 1822 unsigned short get_checksum_mem(char *buff, int bytes) 1823 { 1824 return get_checksum(buff, bytes, 0); 1825 } 1826 1827 1828 unsigned short get_checksum_mem_buffer(struct file_buffer *file_buffer) 1829 { 1830 if(file_buffer == NULL) 1831 return 0; 1832 else 1833 return get_checksum(file_buffer->data, file_buffer->size, 0); 1834 } 1835 1836 1837 #define DUP_HASH(a) (a & 0xffff) 1838 void add_file(long long start, long long file_size, long long file_bytes, 1839 unsigned int *block_listp, int blocks, unsigned int fragment, 1840 int offset, int bytes) 1841 { 1842 struct fragment *frg; 1843 unsigned int *block_list = block_listp; 1844 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1845 struct append_file *append_file; 1846 struct file_info *file; 1847 1848 if(!duplicate_checking || file_size == 0) 1849 return; 1850 1851 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) { 1852 if(file_size != dupl_ptr->file_size) 1853 continue; 1854 if(blocks != 0 && start != dupl_ptr->start) 1855 continue; 1856 if(fragment != dupl_ptr->fragment->index) 1857 continue; 1858 if(fragment != SQUASHFS_INVALID_FRAG && (offset != 1859 dupl_ptr->fragment->offset || bytes != 1860 dupl_ptr->fragment->size)) 1861 continue; 1862 return; 1863 } 1864 1865 frg = malloc(sizeof(struct fragment)); 1866 if(frg == NULL) 1867 MEM_ERROR(); 1868 1869 frg->index = fragment; 1870 frg->offset = offset; 1871 frg->size = bytes; 1872 1873 file = add_non_dup(file_size, file_bytes, block_list, start, frg, 0, 0, 1874 FALSE, FALSE); 1875 1876 if(fragment == SQUASHFS_INVALID_FRAG) 1877 return; 1878 1879 append_file = malloc(sizeof(struct append_file)); 1880 if(append_file == NULL) 1881 MEM_ERROR(); 1882 1883 append_file->file = file; 1884 append_file->next = file_mapping[fragment]; 1885 file_mapping[fragment] = append_file; 1886 } 1887 1888 1889 int pre_duplicate(long long file_size) 1890 { 1891 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1892 1893 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) 1894 if(dupl_ptr->file_size == file_size) 1895 return TRUE; 1896 1897 return FALSE; 1898 } 1899 1900 1901 struct file_info *add_non_dup(long long file_size, long long bytes, 1902 unsigned int *block_list, long long start, struct fragment *fragment, 1903 unsigned short checksum, unsigned short fragment_checksum, 1904 int checksum_flag, int checksum_frag_flag) 1905 { 1906 struct file_info *dupl_ptr = malloc(sizeof(struct file_info)); 1907 1908 if(dupl_ptr == NULL) 1909 MEM_ERROR(); 1910 1911 dupl_ptr->file_size = file_size; 1912 dupl_ptr->bytes = bytes; 1913 dupl_ptr->block_list = block_list; 1914 dupl_ptr->start = start; 1915 dupl_ptr->fragment = fragment; 1916 dupl_ptr->checksum = checksum; 1917 dupl_ptr->fragment_checksum = fragment_checksum; 1918 dupl_ptr->have_frag_checksum = checksum_frag_flag; 1919 dupl_ptr->have_checksum = checksum_flag; 1920 1921 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1922 pthread_mutex_lock(&dup_mutex); 1923 dupl_ptr->next = dupl[DUP_HASH(file_size)]; 1924 dupl[DUP_HASH(file_size)] = dupl_ptr; 1925 dup_files ++; 1926 pthread_cleanup_pop(1); 1927 1928 return dupl_ptr; 1929 } 1930 1931 1932 struct fragment *frag_duplicate(struct file_buffer *file_buffer, char *dont_put) 1933 { 1934 struct file_info *dupl_ptr; 1935 struct file_buffer *buffer; 1936 struct file_info *dupl_start = file_buffer->dupl_start; 1937 long long file_size = file_buffer->file_size; 1938 unsigned short checksum = file_buffer->checksum; 1939 int res; 1940 1941 if(file_buffer->duplicate) { 1942 TRACE("Found duplicate file, fragment %d, size %d, offset %d, " 1943 "checksum 0x%x\n", dupl_start->fragment->index, 1944 file_size, dupl_start->fragment->offset, checksum); 1945 *dont_put = TRUE; 1946 return dupl_start->fragment; 1947 } else { 1948 *dont_put = FALSE; 1949 dupl_ptr = dupl[DUP_HASH(file_size)]; 1950 } 1951 1952 for(; dupl_ptr && dupl_ptr != dupl_start; dupl_ptr = dupl_ptr->next) { 1953 if(file_size == dupl_ptr->file_size && file_size == 1954 dupl_ptr->fragment->size) { 1955 if(get_fragment_checksum(dupl_ptr) == checksum) { 1956 buffer = get_fragment(dupl_ptr->fragment); 1957 res = memcmp(file_buffer->data, buffer->data + 1958 dupl_ptr->fragment->offset, file_size); 1959 cache_block_put(buffer); 1960 if(res == 0) 1961 break; 1962 } 1963 } 1964 } 1965 1966 if(!dupl_ptr || dupl_ptr == dupl_start) 1967 return NULL; 1968 1969 TRACE("Found duplicate file, fragment %d, size %d, offset %d, " 1970 "checksum 0x%x\n", dupl_ptr->fragment->index, file_size, 1971 dupl_ptr->fragment->offset, checksum); 1972 1973 return dupl_ptr->fragment; 1974 } 1975 1976 1977 struct file_info *duplicate(long long file_size, long long bytes, 1978 unsigned int **block_list, long long *start, struct fragment **fragment, 1979 struct file_buffer *file_buffer, int blocks, unsigned short checksum, 1980 int checksum_flag) 1981 { 1982 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1983 int frag_bytes = file_buffer ? file_buffer->size : 0; 1984 unsigned short fragment_checksum = file_buffer ? 1985 file_buffer->checksum : 0; 1986 1987 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) 1988 if(file_size == dupl_ptr->file_size && bytes == dupl_ptr->bytes 1989 && frag_bytes == dupl_ptr->fragment->size) { 1990 long long target_start, dup_start = dupl_ptr->start; 1991 int block; 1992 1993 if(memcmp(*block_list, dupl_ptr->block_list, blocks * 1994 sizeof(unsigned int)) != 0) 1995 continue; 1996 1997 if(checksum_flag == FALSE) { 1998 checksum = get_checksum_disk(*start, bytes, 1999 *block_list); 2000 checksum_flag = TRUE; 2001 } 2002 2003 if(!dupl_ptr->have_checksum) { 2004 dupl_ptr->checksum = 2005 get_checksum_disk(dupl_ptr->start, 2006 dupl_ptr->bytes, dupl_ptr->block_list); 2007 dupl_ptr->have_checksum = TRUE; 2008 } 2009 2010 if(checksum != dupl_ptr->checksum || 2011 fragment_checksum != 2012 get_fragment_checksum(dupl_ptr)) 2013 continue; 2014 2015 target_start = *start; 2016 for(block = 0; block < blocks; block ++) { 2017 int size = SQUASHFS_COMPRESSED_SIZE_BLOCK 2018 ((*block_list)[block]); 2019 struct file_buffer *target_buffer = NULL; 2020 struct file_buffer *dup_buffer = NULL; 2021 char *target_data, *dup_data; 2022 int res; 2023 2024 if(size == 0) 2025 continue; 2026 target_buffer = cache_lookup(bwriter_buffer, 2027 target_start); 2028 if(target_buffer) 2029 target_data = target_buffer->data; 2030 else { 2031 target_data = 2032 read_from_disk(target_start, 2033 size); 2034 if(target_data == NULL) { 2035 ERROR("Failed to read data from" 2036 " output filesystem\n"); 2037 BAD_ERROR("Output filesystem" 2038 " corrupted?\n"); 2039 } 2040 } 2041 2042 dup_buffer = cache_lookup(bwriter_buffer, 2043 dup_start); 2044 if(dup_buffer) 2045 dup_data = dup_buffer->data; 2046 else { 2047 dup_data = read_from_disk2(dup_start, 2048 size); 2049 if(dup_data == NULL) { 2050 ERROR("Failed to read data from" 2051 " output filesystem\n"); 2052 BAD_ERROR("Output filesystem" 2053 " corrupted?\n"); 2054 } 2055 } 2056 2057 res = memcmp(target_data, dup_data, size); 2058 cache_block_put(target_buffer); 2059 cache_block_put(dup_buffer); 2060 if(res != 0) 2061 break; 2062 target_start += size; 2063 dup_start += size; 2064 } 2065 if(block == blocks) { 2066 struct file_buffer *frag_buffer = 2067 get_fragment(dupl_ptr->fragment); 2068 2069 if(frag_bytes == 0 || 2070 memcmp(file_buffer->data, 2071 frag_buffer->data + 2072 dupl_ptr->fragment->offset, 2073 frag_bytes) == 0) { 2074 TRACE("Found duplicate file, start " 2075 "0x%llx, size %lld, checksum " 2076 "0x%x, fragment %d, size %d, " 2077 "offset %d, checksum 0x%x\n", 2078 dupl_ptr->start, 2079 dupl_ptr->bytes, 2080 dupl_ptr->checksum, 2081 dupl_ptr->fragment->index, 2082 frag_bytes, 2083 dupl_ptr->fragment->offset, 2084 fragment_checksum); 2085 *block_list = dupl_ptr->block_list; 2086 *start = dupl_ptr->start; 2087 *fragment = dupl_ptr->fragment; 2088 cache_block_put(frag_buffer); 2089 return 0; 2090 } 2091 cache_block_put(frag_buffer); 2092 } 2093 } 2094 2095 2096 return add_non_dup(file_size, bytes, *block_list, *start, *fragment, 2097 checksum, fragment_checksum, checksum_flag, TRUE); 2098 } 2099 2100 2101 static inline int is_fragment(struct inode_info *inode) 2102 { 2103 off_t file_size = inode->buf.st_size; 2104 2105 /* 2106 * If this block is to be compressed differently to the 2107 * fragment compression then it cannot be a fragment 2108 */ 2109 if(inode->noF != noF) 2110 return FALSE; 2111 2112 return !inode->no_fragments && file_size && (file_size < block_size || 2113 (inode->always_use_fragments && file_size & (block_size - 1))); 2114 } 2115 2116 2117 void put_file_buffer(struct file_buffer *file_buffer) 2118 { 2119 /* 2120 * Decide where to send the file buffer: 2121 * - compressible non-fragment blocks go to the deflate threads, 2122 * - fragments go to the process fragment threads, 2123 * - all others go directly to the main thread 2124 */ 2125 if(file_buffer->error) { 2126 file_buffer->fragment = 0; 2127 seq_queue_put(to_main, file_buffer); 2128 } else if (file_buffer->file_size == 0) 2129 seq_queue_put(to_main, file_buffer); 2130 else if(file_buffer->fragment) 2131 queue_put(to_process_frag, file_buffer); 2132 else 2133 queue_put(to_deflate, file_buffer); 2134 } 2135 2136 2137 static int seq = 0; 2138 void reader_read_process(struct dir_ent *dir_ent) 2139 { 2140 long long bytes = 0; 2141 struct inode_info *inode = dir_ent->inode; 2142 struct file_buffer *prev_buffer = NULL, *file_buffer; 2143 int status, byte, res, child; 2144 int file = pseudo_exec_file(get_pseudo_file(inode->pseudo_id), &child); 2145 2146 if(!file) { 2147 file_buffer = cache_get_nohash(reader_buffer); 2148 file_buffer->sequence = seq ++; 2149 goto read_err; 2150 } 2151 2152 while(1) { 2153 file_buffer = cache_get_nohash(reader_buffer); 2154 file_buffer->sequence = seq ++; 2155 file_buffer->noD = inode->noD; 2156 2157 byte = read_bytes(file, file_buffer->data, block_size); 2158 if(byte == -1) 2159 goto read_err2; 2160 2161 file_buffer->size = byte; 2162 file_buffer->file_size = -1; 2163 file_buffer->error = FALSE; 2164 file_buffer->fragment = FALSE; 2165 bytes += byte; 2166 2167 if(byte == 0) 2168 break; 2169 2170 /* 2171 * Update progress bar size. This is done 2172 * on every block rather than waiting for all blocks to be 2173 * read incase write_file_process() is running in parallel 2174 * with this. Otherwise the current progress bar position 2175 * may get ahead of the progress bar size. 2176 */ 2177 progress_bar_size(1); 2178 2179 if(prev_buffer) 2180 put_file_buffer(prev_buffer); 2181 prev_buffer = file_buffer; 2182 } 2183 2184 /* 2185 * Update inode file size now that the size of the dynamic pseudo file 2186 * is known. This is needed for the -info option. 2187 */ 2188 inode->buf.st_size = bytes; 2189 2190 res = waitpid(child, &status, 0); 2191 close(file); 2192 2193 if(res == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) 2194 goto read_err; 2195 2196 if(prev_buffer == NULL) 2197 prev_buffer = file_buffer; 2198 else { 2199 cache_block_put(file_buffer); 2200 seq --; 2201 } 2202 prev_buffer->file_size = bytes; 2203 prev_buffer->fragment = is_fragment(inode); 2204 put_file_buffer(prev_buffer); 2205 2206 return; 2207 2208 read_err2: 2209 close(file); 2210 read_err: 2211 if(prev_buffer) { 2212 cache_block_put(file_buffer); 2213 seq --; 2214 file_buffer = prev_buffer; 2215 } 2216 file_buffer->error = TRUE; 2217 put_file_buffer(file_buffer); 2218 } 2219 2220 2221 void reader_read_file(struct dir_ent *dir_ent) 2222 { 2223 struct stat *buf = &dir_ent->inode->buf, buf2; 2224 struct file_buffer *file_buffer; 2225 int blocks, file, res; 2226 long long bytes, read_size; 2227 struct inode_info *inode = dir_ent->inode; 2228 2229 if(inode->read) 2230 return; 2231 2232 inode->read = TRUE; 2233 again: 2234 bytes = 0; 2235 read_size = buf->st_size; 2236 blocks = (read_size + block_size - 1) >> block_log; 2237 2238 file = open(pathname_reader(dir_ent), O_RDONLY); 2239 if(file == -1) { 2240 file_buffer = cache_get_nohash(reader_buffer); 2241 file_buffer->sequence = seq ++; 2242 goto read_err2; 2243 } 2244 2245 do { 2246 file_buffer = cache_get_nohash(reader_buffer); 2247 file_buffer->file_size = read_size; 2248 file_buffer->sequence = seq ++; 2249 file_buffer->noD = inode->noD; 2250 file_buffer->error = FALSE; 2251 2252 /* 2253 * Always try to read block_size bytes from the file rather 2254 * than expected bytes (which will be less than the block_size 2255 * at the file tail) to check that the file hasn't grown 2256 * since being stated. If it is longer (or shorter) than 2257 * expected, then restat, and try again. Note the special 2258 * case where the file is an exact multiple of the block_size 2259 * is dealt with later. 2260 */ 2261 file_buffer->size = read_bytes(file, file_buffer->data, 2262 block_size); 2263 if(file_buffer->size == -1) 2264 goto read_err; 2265 2266 bytes += file_buffer->size; 2267 2268 if(blocks > 1) { 2269 /* non-tail block should be exactly block_size */ 2270 if(file_buffer->size < block_size) 2271 goto restat; 2272 2273 file_buffer->fragment = FALSE; 2274 put_file_buffer(file_buffer); 2275 } 2276 } while(-- blocks > 0); 2277 2278 /* Overall size including tail should match */ 2279 if(read_size != bytes) 2280 goto restat; 2281 2282 if(read_size && read_size % block_size == 0) { 2283 /* 2284 * Special case where we've not tried to read past the end of 2285 * the file. We expect to get EOF, i.e. the file isn't larger 2286 * than we expect. 2287 */ 2288 char buffer; 2289 int res; 2290 2291 res = read_bytes(file, &buffer, 1); 2292 if(res == -1) 2293 goto read_err; 2294 2295 if(res != 0) 2296 goto restat; 2297 } 2298 2299 file_buffer->fragment = is_fragment(inode); 2300 put_file_buffer(file_buffer); 2301 2302 close(file); 2303 2304 return; 2305 2306 restat: 2307 res = fstat(file, &buf2); 2308 if(res == -1) { 2309 ERROR("Cannot stat dir/file %s because %s\n", 2310 pathname_reader(dir_ent), strerror(errno)); 2311 goto read_err; 2312 } 2313 2314 if(read_size != buf2.st_size) { 2315 close(file); 2316 memcpy(buf, &buf2, sizeof(struct stat)); 2317 file_buffer->error = 2; 2318 put_file_buffer(file_buffer); 2319 goto again; 2320 } 2321 read_err: 2322 close(file); 2323 read_err2: 2324 file_buffer->error = TRUE; 2325 put_file_buffer(file_buffer); 2326 } 2327 2328 2329 void reader_scan(struct dir_info *dir) { 2330 struct dir_ent *dir_ent = dir->list; 2331 2332 for(; dir_ent; dir_ent = dir_ent->next) { 2333 struct stat *buf = &dir_ent->inode->buf; 2334 if(dir_ent->inode->root_entry) 2335 continue; 2336 2337 if(IS_PSEUDO_PROCESS(dir_ent->inode)) { 2338 reader_read_process(dir_ent); 2339 continue; 2340 } 2341 2342 switch(buf->st_mode & S_IFMT) { 2343 case S_IFREG: 2344 reader_read_file(dir_ent); 2345 break; 2346 case S_IFDIR: 2347 reader_scan(dir_ent->dir); 2348 break; 2349 } 2350 } 2351 } 2352 2353 2354 void *reader(void *arg) 2355 { 2356 if(!sorted) 2357 reader_scan(queue_get(to_reader)); 2358 else { 2359 int i; 2360 struct priority_entry *entry; 2361 2362 queue_get(to_reader); 2363 for(i = 65535; i >= 0; i--) 2364 for(entry = priority_list[i]; entry; 2365 entry = entry->next) 2366 reader_read_file(entry->dir); 2367 } 2368 2369 pthread_exit(NULL); 2370 } 2371 2372 2373 void *writer(void *arg) 2374 { 2375 while(1) { 2376 struct file_buffer *file_buffer = queue_get(to_writer); 2377 off_t off; 2378 2379 if(file_buffer == NULL) { 2380 queue_put(from_writer, NULL); 2381 continue; 2382 } 2383 2384 off = file_buffer->block; 2385 2386 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 2387 pthread_mutex_lock(&pos_mutex); 2388 2389 if(lseek(fd, off, SEEK_SET) == -1) { 2390 ERROR("writer: Lseek on destination failed because " 2391 "%s, offset=0x%llx\n", strerror(errno), off); 2392 BAD_ERROR("Probably out of space on output " 2393 "%s\n", block_device ? "block device" : 2394 "filesystem"); 2395 } 2396 2397 if(write_bytes(fd, file_buffer->data, 2398 file_buffer->size) == -1) 2399 BAD_ERROR("Failed to write to output %s\n", 2400 block_device ? "block device" : "filesystem"); 2401 2402 pthread_cleanup_pop(1); 2403 2404 cache_block_put(file_buffer); 2405 } 2406 } 2407 2408 2409 int all_zero(struct file_buffer *file_buffer) 2410 { 2411 int i; 2412 long entries = file_buffer->size / sizeof(long); 2413 long *p = (long *) file_buffer->data; 2414 2415 for(i = 0; i < entries && p[i] == 0; i++); 2416 2417 if(i == entries) { 2418 for(i = file_buffer->size & ~(sizeof(long) - 1); 2419 i < file_buffer->size && file_buffer->data[i] == 0; 2420 i++); 2421 2422 return i == file_buffer->size; 2423 } 2424 2425 return 0; 2426 } 2427 2428 2429 void *deflator(void *arg) 2430 { 2431 struct file_buffer *write_buffer = cache_get_nohash(bwriter_buffer); 2432 void *stream = NULL; 2433 int res; 2434 2435 res = compressor_init(comp, &stream, block_size, 1); 2436 if(res) 2437 BAD_ERROR("deflator:: compressor_init failed\n"); 2438 2439 while(1) { 2440 struct file_buffer *file_buffer = queue_get(to_deflate); 2441 2442 if(sparse_files && all_zero(file_buffer)) { 2443 file_buffer->c_byte = 0; 2444 seq_queue_put(to_main, file_buffer); 2445 } else { 2446 write_buffer->c_byte = mangle2(stream, 2447 write_buffer->data, file_buffer->data, 2448 file_buffer->size, block_size, 2449 file_buffer->noD, 1); 2450 write_buffer->sequence = file_buffer->sequence; 2451 write_buffer->file_size = file_buffer->file_size; 2452 write_buffer->block = file_buffer->block; 2453 write_buffer->size = SQUASHFS_COMPRESSED_SIZE_BLOCK 2454 (write_buffer->c_byte); 2455 write_buffer->fragment = FALSE; 2456 write_buffer->error = FALSE; 2457 cache_block_put(file_buffer); 2458 seq_queue_put(to_main, write_buffer); 2459 write_buffer = cache_get_nohash(bwriter_buffer); 2460 } 2461 } 2462 } 2463 2464 2465 void *frag_deflator(void *arg) 2466 { 2467 void *stream = NULL; 2468 int res; 2469 2470 res = compressor_init(comp, &stream, block_size, 1); 2471 if(res) 2472 BAD_ERROR("frag_deflator:: compressor_init failed\n"); 2473 2474 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 2475 2476 while(1) { 2477 int c_byte, compressed_size; 2478 struct file_buffer *file_buffer = queue_get(to_frag); 2479 struct file_buffer *write_buffer = 2480 cache_get(fwriter_buffer, file_buffer->block); 2481 2482 c_byte = mangle2(stream, write_buffer->data, file_buffer->data, 2483 file_buffer->size, block_size, noF, 1); 2484 compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); 2485 write_buffer->size = compressed_size; 2486 pthread_mutex_lock(&fragment_mutex); 2487 if(fragments_locked == FALSE) { 2488 fragment_table[file_buffer->block].size = c_byte; 2489 fragment_table[file_buffer->block].start_block = bytes; 2490 write_buffer->block = bytes; 2491 bytes += compressed_size; 2492 fragments_outstanding --; 2493 queue_put(to_writer, write_buffer); 2494 pthread_mutex_unlock(&fragment_mutex); 2495 TRACE("Writing fragment %lld, uncompressed size %d, " 2496 "compressed size %d\n", file_buffer->block, 2497 file_buffer->size, compressed_size); 2498 } else { 2499 add_pending_fragment(write_buffer, c_byte, 2500 file_buffer->block); 2501 pthread_mutex_unlock(&fragment_mutex); 2502 } 2503 cache_block_put(file_buffer); 2504 } 2505 2506 pthread_cleanup_pop(0); 2507 } 2508 2509 2510 struct file_buffer *get_file_buffer() 2511 { 2512 struct file_buffer *file_buffer = seq_queue_get(to_main); 2513 2514 return file_buffer; 2515 } 2516 2517 2518 void write_file_empty(squashfs_inode *inode, struct dir_ent *dir_ent, 2519 struct file_buffer *file_buffer, int *duplicate_file) 2520 { 2521 file_count ++; 2522 *duplicate_file = FALSE; 2523 cache_block_put(file_buffer); 2524 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, 0, 0, 0, 2525 NULL, &empty_fragment, NULL, 0); 2526 } 2527 2528 2529 void write_file_frag(squashfs_inode *inode, struct dir_ent *dir_ent, 2530 struct file_buffer *file_buffer, int *duplicate_file) 2531 { 2532 int size = file_buffer->file_size; 2533 struct fragment *fragment; 2534 unsigned short checksum = file_buffer->checksum; 2535 char dont_put; 2536 2537 fragment = frag_duplicate(file_buffer, &dont_put); 2538 *duplicate_file = !fragment; 2539 if(!fragment) { 2540 fragment = get_and_fill_fragment(file_buffer, dir_ent); 2541 if(duplicate_checking) 2542 add_non_dup(size, 0, NULL, 0, fragment, 0, checksum, 2543 TRUE, TRUE); 2544 } 2545 2546 if(dont_put) 2547 free(file_buffer); 2548 else 2549 cache_block_put(file_buffer); 2550 2551 total_bytes += size; 2552 file_count ++; 2553 2554 inc_progress_bar(); 2555 2556 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, size, 0, 2557 0, NULL, fragment, NULL, 0); 2558 2559 if(!duplicate_checking) 2560 free_fragment(fragment); 2561 } 2562 2563 2564 int write_file_process(squashfs_inode *inode, struct dir_ent *dir_ent, 2565 struct file_buffer *read_buffer, int *duplicate_file) 2566 { 2567 long long read_size, file_bytes, start; 2568 struct fragment *fragment; 2569 unsigned int *block_list = NULL; 2570 int block = 0, status; 2571 long long sparse = 0; 2572 struct file_buffer *fragment_buffer = NULL; 2573 2574 *duplicate_file = FALSE; 2575 2576 lock_fragments(); 2577 2578 file_bytes = 0; 2579 start = bytes; 2580 while (1) { 2581 read_size = read_buffer->file_size; 2582 if(read_buffer->fragment) 2583 fragment_buffer = read_buffer; 2584 else { 2585 block_list = realloc(block_list, (block + 1) * 2586 sizeof(unsigned int)); 2587 if(block_list == NULL) 2588 MEM_ERROR(); 2589 block_list[block ++] = read_buffer->c_byte; 2590 if(read_buffer->c_byte) { 2591 read_buffer->block = bytes; 2592 bytes += read_buffer->size; 2593 cache_hash(read_buffer, read_buffer->block); 2594 file_bytes += read_buffer->size; 2595 queue_put(to_writer, read_buffer); 2596 } else { 2597 sparse += read_buffer->size; 2598 cache_block_put(read_buffer); 2599 } 2600 } 2601 inc_progress_bar(); 2602 2603 if(read_size != -1) 2604 break; 2605 2606 read_buffer = get_file_buffer(); 2607 if(read_buffer->error) 2608 goto read_err; 2609 } 2610 2611 unlock_fragments(); 2612 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2613 2614 if(duplicate_checking) 2615 add_non_dup(read_size, file_bytes, block_list, start, fragment, 2616 0, fragment_buffer ? fragment_buffer->checksum : 0, 2617 FALSE, TRUE); 2618 cache_block_put(fragment_buffer); 2619 file_count ++; 2620 total_bytes += read_size; 2621 2622 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, 2623 block, block_list, fragment, NULL, sparse); 2624 2625 if(duplicate_checking == FALSE) { 2626 free(block_list); 2627 free_fragment(fragment); 2628 } 2629 2630 return 0; 2631 2632 read_err: 2633 dec_progress_bar(block); 2634 status = read_buffer->error; 2635 bytes = start; 2636 if(!block_device) { 2637 int res; 2638 2639 queue_put(to_writer, NULL); 2640 if(queue_get(from_writer) != 0) 2641 EXIT_MKSQUASHFS(); 2642 res = ftruncate(fd, bytes); 2643 if(res != 0) 2644 BAD_ERROR("Failed to truncate dest file because %s\n", 2645 strerror(errno)); 2646 } 2647 unlock_fragments(); 2648 free(block_list); 2649 cache_block_put(read_buffer); 2650 return status; 2651 } 2652 2653 2654 int write_file_blocks_dup(squashfs_inode *inode, struct dir_ent *dir_ent, 2655 struct file_buffer *read_buffer, int *duplicate_file) 2656 { 2657 int block, thresh; 2658 long long read_size = read_buffer->file_size; 2659 long long file_bytes, dup_start, start; 2660 struct fragment *fragment; 2661 struct file_info *dupl_ptr; 2662 int blocks = (read_size + block_size - 1) >> block_log; 2663 unsigned int *block_list, *block_listp; 2664 struct file_buffer **buffer_list; 2665 int status; 2666 long long sparse = 0; 2667 struct file_buffer *fragment_buffer = NULL; 2668 2669 block_list = malloc(blocks * sizeof(unsigned int)); 2670 if(block_list == NULL) 2671 MEM_ERROR(); 2672 block_listp = block_list; 2673 2674 buffer_list = malloc(blocks * sizeof(struct file_buffer *)); 2675 if(buffer_list == NULL) 2676 MEM_ERROR(); 2677 2678 lock_fragments(); 2679 2680 file_bytes = 0; 2681 start = dup_start = bytes; 2682 thresh = blocks > bwriter_size ? blocks - bwriter_size : 0; 2683 2684 for(block = 0; block < blocks;) { 2685 if(read_buffer->fragment) { 2686 block_list[block] = 0; 2687 buffer_list[block] = NULL; 2688 fragment_buffer = read_buffer; 2689 blocks = read_size >> block_log; 2690 } else { 2691 block_list[block] = read_buffer->c_byte; 2692 2693 if(read_buffer->c_byte) { 2694 read_buffer->block = bytes; 2695 bytes += read_buffer->size; 2696 file_bytes += read_buffer->size; 2697 cache_hash(read_buffer, read_buffer->block); 2698 if(block < thresh) { 2699 buffer_list[block] = NULL; 2700 queue_put(to_writer, read_buffer); 2701 } else 2702 buffer_list[block] = read_buffer; 2703 } else { 2704 buffer_list[block] = NULL; 2705 sparse += read_buffer->size; 2706 cache_block_put(read_buffer); 2707 } 2708 } 2709 inc_progress_bar(); 2710 2711 if(++block < blocks) { 2712 read_buffer = get_file_buffer(); 2713 if(read_buffer->error) 2714 goto read_err; 2715 } 2716 } 2717 2718 dupl_ptr = duplicate(read_size, file_bytes, &block_listp, &dup_start, 2719 &fragment, fragment_buffer, blocks, 0, FALSE); 2720 2721 if(dupl_ptr) { 2722 *duplicate_file = FALSE; 2723 for(block = thresh; block < blocks; block ++) 2724 if(buffer_list[block]) 2725 queue_put(to_writer, buffer_list[block]); 2726 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2727 dupl_ptr->fragment = fragment; 2728 } else { 2729 *duplicate_file = TRUE; 2730 for(block = thresh; block < blocks; block ++) 2731 cache_block_put(buffer_list[block]); 2732 bytes = start; 2733 if(thresh && !block_device) { 2734 int res; 2735 2736 queue_put(to_writer, NULL); 2737 if(queue_get(from_writer) != 0) 2738 EXIT_MKSQUASHFS(); 2739 res = ftruncate(fd, bytes); 2740 if(res != 0) 2741 BAD_ERROR("Failed to truncate dest file because" 2742 " %s\n", strerror(errno)); 2743 } 2744 } 2745 2746 unlock_fragments(); 2747 cache_block_put(fragment_buffer); 2748 free(buffer_list); 2749 file_count ++; 2750 total_bytes += read_size; 2751 2752 /* 2753 * sparse count is needed to ensure squashfs correctly reports a 2754 * a smaller block count on stat calls to sparse files. This is 2755 * to ensure intelligent applications like cp correctly handle the 2756 * file as a sparse file. If the file in the original filesystem isn't 2757 * stored as a sparse file then still store it sparsely in squashfs, but 2758 * report it as non-sparse on stat calls to preserve semantics 2759 */ 2760 if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) 2761 sparse = 0; 2762 2763 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, 2764 dup_start, blocks, block_listp, fragment, NULL, sparse); 2765 2766 if(*duplicate_file == TRUE) 2767 free(block_list); 2768 2769 return 0; 2770 2771 read_err: 2772 dec_progress_bar(block); 2773 status = read_buffer->error; 2774 bytes = start; 2775 if(thresh && !block_device) { 2776 int res; 2777 2778 queue_put(to_writer, NULL); 2779 if(queue_get(from_writer) != 0) 2780 EXIT_MKSQUASHFS(); 2781 res = ftruncate(fd, bytes); 2782 if(res != 0) 2783 BAD_ERROR("Failed to truncate dest file because %s\n", 2784 strerror(errno)); 2785 } 2786 unlock_fragments(); 2787 for(blocks = thresh; blocks < block; blocks ++) 2788 cache_block_put(buffer_list[blocks]); 2789 free(buffer_list); 2790 free(block_list); 2791 cache_block_put(read_buffer); 2792 return status; 2793 } 2794 2795 2796 int write_file_blocks(squashfs_inode *inode, struct dir_ent *dir_ent, 2797 struct file_buffer *read_buffer, int *dup) 2798 { 2799 long long read_size = read_buffer->file_size; 2800 long long file_bytes, start; 2801 struct fragment *fragment; 2802 unsigned int *block_list; 2803 int block, status; 2804 int blocks = (read_size + block_size - 1) >> block_log; 2805 long long sparse = 0; 2806 struct file_buffer *fragment_buffer = NULL; 2807 2808 if(pre_duplicate(read_size)) 2809 return write_file_blocks_dup(inode, dir_ent, read_buffer, dup); 2810 2811 *dup = FALSE; 2812 2813 block_list = malloc(blocks * sizeof(unsigned int)); 2814 if(block_list == NULL) 2815 MEM_ERROR(); 2816 2817 lock_fragments(); 2818 2819 file_bytes = 0; 2820 /* ANDROID CHANGES START*/ 2821 #ifdef ANDROID 2822 if (align_4k_blocks && bytes % 4096) { 2823 bytes += 4096 - (bytes % 4096); 2824 } 2825 #endif 2826 /* ANDROID CHANGES END */ 2827 start = bytes; 2828 for(block = 0; block < blocks;) { 2829 if(read_buffer->fragment) { 2830 block_list[block] = 0; 2831 fragment_buffer = read_buffer; 2832 blocks = read_size >> block_log; 2833 } else { 2834 block_list[block] = read_buffer->c_byte; 2835 if(read_buffer->c_byte) { 2836 read_buffer->block = bytes; 2837 bytes += read_buffer->size; 2838 cache_hash(read_buffer, read_buffer->block); 2839 file_bytes += read_buffer->size; 2840 queue_put(to_writer, read_buffer); 2841 } else { 2842 sparse += read_buffer->size; 2843 cache_block_put(read_buffer); 2844 } 2845 } 2846 inc_progress_bar(); 2847 2848 if(++block < blocks) { 2849 read_buffer = get_file_buffer(); 2850 if(read_buffer->error) 2851 goto read_err; 2852 } 2853 } 2854 2855 unlock_fragments(); 2856 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2857 2858 if(duplicate_checking) 2859 add_non_dup(read_size, file_bytes, block_list, start, fragment, 2860 0, fragment_buffer ? fragment_buffer->checksum : 0, 2861 FALSE, TRUE); 2862 cache_block_put(fragment_buffer); 2863 file_count ++; 2864 total_bytes += read_size; 2865 2866 /* 2867 * sparse count is needed to ensure squashfs correctly reports a 2868 * a smaller block count on stat calls to sparse files. This is 2869 * to ensure intelligent applications like cp correctly handle the 2870 * file as a sparse file. If the file in the original filesystem isn't 2871 * stored as a sparse file then still store it sparsely in squashfs, but 2872 * report it as non-sparse on stat calls to preserve semantics 2873 */ 2874 if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) 2875 sparse = 0; 2876 2877 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, 2878 blocks, block_list, fragment, NULL, sparse); 2879 2880 if(duplicate_checking == FALSE) { 2881 free(block_list); 2882 free_fragment(fragment); 2883 } 2884 2885 return 0; 2886 2887 read_err: 2888 dec_progress_bar(block); 2889 status = read_buffer->error; 2890 bytes = start; 2891 if(!block_device) { 2892 int res; 2893 2894 queue_put(to_writer, NULL); 2895 if(queue_get(from_writer) != 0) 2896 EXIT_MKSQUASHFS(); 2897 res = ftruncate(fd, bytes); 2898 if(res != 0) 2899 BAD_ERROR("Failed to truncate dest file because %s\n", 2900 strerror(errno)); 2901 } 2902 unlock_fragments(); 2903 free(block_list); 2904 cache_block_put(read_buffer); 2905 return status; 2906 } 2907 2908 2909 void write_file(squashfs_inode *inode, struct dir_ent *dir, int *dup) 2910 { 2911 int status; 2912 struct file_buffer *read_buffer; 2913 2914 again: 2915 read_buffer = get_file_buffer(); 2916 status = read_buffer->error; 2917 2918 if(status) 2919 cache_block_put(read_buffer); 2920 else if(read_buffer->file_size == -1) 2921 status = write_file_process(inode, dir, read_buffer, dup); 2922 else if(read_buffer->file_size == 0) 2923 write_file_empty(inode, dir, read_buffer, dup); 2924 else if(read_buffer->fragment && read_buffer->c_byte) 2925 write_file_frag(inode, dir, read_buffer, dup); 2926 else 2927 status = write_file_blocks(inode, dir, read_buffer, dup); 2928 2929 if(status == 2) { 2930 ERROR("File %s changed size while reading filesystem, " 2931 "attempting to re-read\n", pathname(dir)); 2932 goto again; 2933 } else if(status == 1) { 2934 ERROR_START("Failed to read file %s", pathname(dir)); 2935 ERROR_EXIT(", creating empty file\n"); 2936 write_file_empty(inode, dir, NULL, dup); 2937 } 2938 } 2939 2940 2941 #define BUFF_SIZE 512 2942 char *name; 2943 char *basename_r(); 2944 2945 char *getbase(char *pathname) 2946 { 2947 static char *b_buffer = NULL; 2948 static int b_size = BUFF_SIZE; 2949 char *result; 2950 2951 if(b_buffer == NULL) { 2952 b_buffer = malloc(b_size); 2953 if(b_buffer == NULL) 2954 MEM_ERROR(); 2955 } 2956 2957 while(1) { 2958 if(*pathname != '/') { 2959 result = getcwd(b_buffer, b_size); 2960 if(result == NULL && errno != ERANGE) 2961 BAD_ERROR("Getcwd failed in getbase\n"); 2962 2963 /* enough room for pathname + "/" + '\0' terminator? */ 2964 if(result && strlen(pathname) + 2 <= 2965 b_size - strlen(b_buffer)) { 2966 strcat(strcat(b_buffer, "/"), pathname); 2967 break; 2968 } 2969 } else if(strlen(pathname) < b_size) { 2970 strcpy(b_buffer, pathname); 2971 break; 2972 } 2973 2974 /* Buffer not large enough, realloc and try again */ 2975 b_buffer = realloc(b_buffer, b_size += BUFF_SIZE); 2976 if(b_buffer == NULL) 2977 MEM_ERROR(); 2978 } 2979 2980 name = b_buffer; 2981 if(((result = basename_r()) == NULL) || (strcmp(result, "..") == 0)) 2982 return NULL; 2983 else 2984 return result; 2985 } 2986 2987 2988 char *basename_r() 2989 { 2990 char *s; 2991 char *p; 2992 int n = 1; 2993 2994 for(;;) { 2995 s = name; 2996 if(*name == '\0') 2997 return NULL; 2998 if(*name != '/') { 2999 while(*name != '\0' && *name != '/') name++; 3000 n = name - s; 3001 } 3002 while(*name == '/') name++; 3003 if(strncmp(s, ".", n) == 0) 3004 continue; 3005 if((*name == '\0') || (strncmp(s, "..", n) == 0) || 3006 ((p = basename_r()) == NULL)) { 3007 s[n] = '\0'; 3008 return s; 3009 } 3010 if(strcmp(p, "..") == 0) 3011 continue; 3012 return p; 3013 } 3014 } 3015 3016 3017 struct inode_info *lookup_inode3(struct stat *buf, int pseudo, int id, 3018 char *symlink, int bytes) 3019 { 3020 int ino_hash = INODE_HASH(buf->st_dev, buf->st_ino); 3021 struct inode_info *inode; 3022 3023 /* 3024 * Look-up inode in hash table, if it already exists we have a 3025 * hard-link, so increment the nlink count and return it. 3026 * Don't do the look-up for directories because we don't hard-link 3027 * directories. 3028 */ 3029 if ((buf->st_mode & S_IFMT) != S_IFDIR) { 3030 for(inode = inode_info[ino_hash]; inode; inode = inode->next) { 3031 if(memcmp(buf, &inode->buf, sizeof(struct stat)) == 0) { 3032 inode->nlink ++; 3033 return inode; 3034 } 3035 } 3036 } 3037 3038 inode = malloc(sizeof(struct inode_info) + bytes); 3039 if(inode == NULL) 3040 MEM_ERROR(); 3041 3042 if(bytes) 3043 memcpy(&inode->symlink, symlink, bytes); 3044 memcpy(&inode->buf, buf, sizeof(struct stat)); 3045 inode->read = FALSE; 3046 inode->root_entry = FALSE; 3047 inode->pseudo_file = pseudo; 3048 inode->pseudo_id = id; 3049 inode->inode = SQUASHFS_INVALID_BLK; 3050 inode->nlink = 1; 3051 inode->inode_number = 0; 3052 3053 /* 3054 * Copy filesystem wide defaults into inode, these filesystem 3055 * wide defaults may be altered on an individual inode basis by 3056 * user specified actions 3057 * 3058 */ 3059 inode->no_fragments = no_fragments; 3060 inode->always_use_fragments = always_use_fragments; 3061 inode->noD = noD; 3062 inode->noF = noF; 3063 3064 inode->next = inode_info[ino_hash]; 3065 inode_info[ino_hash] = inode; 3066 3067 return inode; 3068 } 3069 3070 3071 static inline struct inode_info *lookup_inode2(struct stat *buf, int pseudo, int id) 3072 { 3073 return lookup_inode3(buf, pseudo, id, NULL, 0); 3074 } 3075 3076 3077 static inline struct inode_info *lookup_inode(struct stat *buf) 3078 { 3079 return lookup_inode2(buf, 0, 0); 3080 } 3081 3082 3083 static inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this) 3084 { 3085 if (inode->inode_number == 0) { 3086 inode->inode_number = use_this ? : inode_no ++; 3087 if((inode->buf.st_mode & S_IFMT) == S_IFREG) 3088 progress_bar_size((inode->buf.st_size + block_size - 1) 3089 >> block_log); 3090 } 3091 } 3092 3093 3094 static inline struct dir_ent *create_dir_entry(char *name, char *source_name, 3095 char *nonstandard_pathname, struct dir_info *dir) 3096 { 3097 struct dir_ent *dir_ent = malloc(sizeof(struct dir_ent)); 3098 if(dir_ent == NULL) 3099 MEM_ERROR(); 3100 3101 dir_ent->name = name; 3102 dir_ent->source_name = source_name; 3103 dir_ent->nonstandard_pathname = nonstandard_pathname; 3104 dir_ent->our_dir = dir; 3105 dir_ent->inode = NULL; 3106 dir_ent->next = NULL; 3107 /* ANDROID CHANGES START*/ 3108 #ifdef ANDROID 3109 dir_ent->capabilities = 0; 3110 #endif 3111 /* ANDROID CHANGES END */ 3112 3113 return dir_ent; 3114 } 3115 3116 3117 static inline void add_dir_entry(struct dir_ent *dir_ent, struct dir_info *sub_dir, 3118 struct inode_info *inode_info) 3119 { 3120 struct dir_info *dir = dir_ent->our_dir; 3121 3122 if(sub_dir) 3123 sub_dir->dir_ent = dir_ent; 3124 3125 /* ANDROID CHANGES START*/ 3126 #ifdef ANDROID 3127 if (android_config) { 3128 if (mount_point) { 3129 char *mounted_path; 3130 char *rel_path; 3131 3132 alloc_mounted_path(mount_point, subpathname(dir_ent), &mounted_path); 3133 rel_path = mounted_path; 3134 while (rel_path && *rel_path == '/') 3135 rel_path++; 3136 android_fs_config(fs_config_func, rel_path, &inode_info->buf, target_out_path, &dir_ent->capabilities); 3137 free(mounted_path); 3138 } else { 3139 android_fs_config(fs_config_func, pathname(dir_ent), &inode_info->buf, target_out_path, &dir_ent->capabilities); 3140 } 3141 } 3142 #endif 3143 /* ANDROID CHANGES END */ 3144 3145 dir_ent->inode = inode_info; 3146 dir_ent->dir = sub_dir; 3147 3148 dir_ent->next = dir->list; 3149 dir->list = dir_ent; 3150 dir->count++; 3151 } 3152 3153 static inline void add_dir_entry2(char *name, char *source_name, 3154 char *nonstandard_pathname, struct dir_info *sub_dir, 3155 struct inode_info *inode_info, struct dir_info *dir) 3156 { 3157 struct dir_ent *dir_ent = create_dir_entry(name, source_name, 3158 nonstandard_pathname, dir); 3159 3160 3161 add_dir_entry(dir_ent, sub_dir, inode_info); 3162 } 3163 3164 3165 static inline void free_dir_entry(struct dir_ent *dir_ent) 3166 { 3167 if(dir_ent->name) 3168 free(dir_ent->name); 3169 3170 if(dir_ent->source_name) 3171 free(dir_ent->source_name); 3172 3173 if(dir_ent->nonstandard_pathname) 3174 free(dir_ent->nonstandard_pathname); 3175 3176 /* if this entry has been associated with an inode, then we need 3177 * to update the inode nlink count. Orphaned inodes are harmless, and 3178 * is easier to leave them than go to the bother of deleting them */ 3179 if(dir_ent->inode && !dir_ent->inode->root_entry) 3180 dir_ent->inode->nlink --; 3181 3182 free(dir_ent); 3183 } 3184 3185 3186 static inline void add_excluded(struct dir_info *dir) 3187 { 3188 dir->excluded ++; 3189 } 3190 3191 3192 void dir_scan(squashfs_inode *inode, char *pathname, 3193 struct dir_ent *(_readdir)(struct dir_info *), int progress) 3194 { 3195 struct stat buf; 3196 struct dir_ent *dir_ent; 3197 /* ANDROID CHANGES START*/ 3198 #ifdef ANDROID 3199 uint64_t caps = 0; 3200 #endif 3201 /* ANDROID CHANGES END */ 3202 3203 root_dir = dir_scan1(pathname, "", paths, _readdir, 1); 3204 if(root_dir == NULL) 3205 return; 3206 3207 /* Create root directory dir_ent and associated inode, and connect 3208 * it to the root directory dir_info structure */ 3209 dir_ent = create_dir_entry("", NULL, pathname, 3210 scan1_opendir("", "", 0)); 3211 3212 if(pathname[0] == '\0') { 3213 /* 3214 * dummy top level directory, if multiple sources specified on 3215 * command line 3216 */ 3217 memset(&buf, 0, sizeof(buf)); 3218 buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR; 3219 buf.st_uid = getuid(); 3220 buf.st_gid = getgid(); 3221 buf.st_mtime = time(NULL); 3222 buf.st_dev = 0; 3223 buf.st_ino = 0; 3224 dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0); 3225 } else { 3226 if(lstat(pathname, &buf) == -1) 3227 /* source directory has disappeared? */ 3228 BAD_ERROR("Cannot stat source directory %s because %s\n", 3229 pathname, strerror(errno)); 3230 /* ANDROID CHANGES START*/ 3231 #ifdef ANDROID 3232 if (android_config) { 3233 if (mount_point) 3234 android_fs_config(fs_config_func, mount_point, &buf, target_out_path, &caps); 3235 else 3236 android_fs_config(fs_config_func, pathname, &buf, target_out_path, &caps); 3237 } 3238 #endif 3239 /* ANDROID CHANGES END */ 3240 dir_ent->inode = lookup_inode(&buf); 3241 } 3242 3243 /* ANDROID CHANGES START*/ 3244 #ifdef ANDROID 3245 dir_ent->capabilities = caps; 3246 #endif 3247 /* ANDROID CHANGES END */ 3248 3249 dir_ent->dir = root_dir; 3250 root_dir->dir_ent = dir_ent; 3251 3252 /* 3253 * Process most actions and any pseudo files 3254 */ 3255 if(actions() || get_pseudo()) 3256 dir_scan2(root_dir, get_pseudo()); 3257 3258 /* 3259 * Process move actions 3260 */ 3261 if(move_actions()) { 3262 dir_scan3(root_dir); 3263 do_move_actions(); 3264 } 3265 3266 /* 3267 * Process prune actions 3268 */ 3269 if(prune_actions()) 3270 dir_scan4(root_dir); 3271 3272 /* 3273 * Process empty actions 3274 */ 3275 if(empty_actions()) 3276 dir_scan5(root_dir); 3277 3278 /* 3279 * Sort directories and compute the inode numbers 3280 */ 3281 dir_scan6(root_dir); 3282 3283 alloc_inode_no(dir_ent->inode, root_inode_number); 3284 3285 eval_actions(root_dir, dir_ent); 3286 3287 if(sorted) 3288 generate_file_priorities(root_dir, 0, 3289 &root_dir->dir_ent->inode->buf); 3290 3291 if(appending) { 3292 sigset_t sigmask; 3293 3294 restore_thread = init_restore_thread(); 3295 sigemptyset(&sigmask); 3296 sigaddset(&sigmask, SIGINT); 3297 sigaddset(&sigmask, SIGTERM); 3298 sigaddset(&sigmask, SIGUSR1); 3299 if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1) 3300 BAD_ERROR("Failed to set signal mask\n"); 3301 write_destination(fd, SQUASHFS_START, 4, "\0\0\0\0"); 3302 } 3303 3304 queue_put(to_reader, root_dir); 3305 3306 set_progressbar_state(progress); 3307 3308 if(sorted) 3309 sort_files_and_write(root_dir); 3310 3311 dir_scan7(inode, root_dir); 3312 dir_ent->inode->inode = *inode; 3313 dir_ent->inode->type = SQUASHFS_DIR_TYPE; 3314 } 3315 3316 3317 /* 3318 * dir_scan1 routines... 3319 * These scan the source directories into memory for processing. 3320 * Exclude actions are processed here (in contrast to the other actions) 3321 * because they affect what is scanned. 3322 */ 3323 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth) 3324 { 3325 struct dir_info *dir; 3326 3327 dir = malloc(sizeof(struct dir_info)); 3328 if(dir == NULL) 3329 MEM_ERROR(); 3330 3331 if(pathname[0] != '\0') { 3332 dir->linuxdir = opendir(pathname); 3333 if(dir->linuxdir == NULL) { 3334 free(dir); 3335 return NULL; 3336 } 3337 } 3338 3339 dir->pathname = strdup(pathname); 3340 dir->subpath = strdup(subpath); 3341 dir->count = 0; 3342 dir->directory_count = 0; 3343 dir->dir_is_ldir = TRUE; 3344 dir->list = NULL; 3345 dir->depth = depth; 3346 dir->excluded = 0; 3347 3348 return dir; 3349 } 3350 3351 3352 struct dir_ent *scan1_encomp_readdir(struct dir_info *dir) 3353 { 3354 static int index = 0; 3355 3356 if(dir->count < old_root_entries) { 3357 int i; 3358 3359 for(i = 0; i < old_root_entries; i++) { 3360 if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) 3361 dir->directory_count ++; 3362 add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, 3363 &old_root_entry[i].inode, dir); 3364 } 3365 } 3366 3367 while(index < source) { 3368 char *basename = NULL; 3369 char *dir_name = getbase(source_path[index]); 3370 int pass = 1, res; 3371 3372 if(dir_name == NULL) { 3373 ERROR_START("Bad source directory %s", 3374 source_path[index]); 3375 ERROR_EXIT(" - skipping ...\n"); 3376 index ++; 3377 continue; 3378 } 3379 dir_name = strdup(dir_name); 3380 for(;;) { 3381 struct dir_ent *dir_ent = dir->list; 3382 3383 for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; 3384 dir_ent = dir_ent->next); 3385 if(dir_ent == NULL) 3386 break; 3387 ERROR("Source directory entry %s already used! - trying" 3388 " ", dir_name); 3389 if(pass == 1) 3390 basename = dir_name; 3391 else 3392 free(dir_name); 3393 res = asprintf(&dir_name, "%s_%d", basename, pass++); 3394 if(res == -1) 3395 BAD_ERROR("asprintf failed in " 3396 "scan1_encomp_readdir\n"); 3397 ERROR("%s\n", dir_name); 3398 } 3399 return create_dir_entry(dir_name, basename, 3400 strdup(source_path[index ++]), dir); 3401 } 3402 return NULL; 3403 } 3404 3405 3406 struct dir_ent *scan1_single_readdir(struct dir_info *dir) 3407 { 3408 struct dirent *d_name; 3409 int i; 3410 3411 if(dir->count < old_root_entries) { 3412 for(i = 0; i < old_root_entries; i++) { 3413 if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) 3414 dir->directory_count ++; 3415 add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, 3416 &old_root_entry[i].inode, dir); 3417 } 3418 } 3419 3420 if((d_name = readdir(dir->linuxdir)) != NULL) { 3421 char *basename = NULL; 3422 char *dir_name = strdup(d_name->d_name); 3423 int pass = 1, res; 3424 3425 for(;;) { 3426 struct dir_ent *dir_ent = dir->list; 3427 3428 for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; 3429 dir_ent = dir_ent->next); 3430 if(dir_ent == NULL) 3431 break; 3432 ERROR("Source directory entry %s already used! - trying" 3433 " ", dir_name); 3434 if (pass == 1) 3435 basename = dir_name; 3436 else 3437 free(dir_name); 3438 res = asprintf(&dir_name, "%s_%d", d_name->d_name, pass++); 3439 if(res == -1) 3440 BAD_ERROR("asprintf failed in " 3441 "scan1_single_readdir\n"); 3442 ERROR("%s\n", dir_name); 3443 } 3444 return create_dir_entry(dir_name, basename, NULL, dir); 3445 } 3446 3447 return NULL; 3448 } 3449 3450 3451 struct dir_ent *scan1_readdir(struct dir_info *dir) 3452 { 3453 struct dirent *d_name = readdir(dir->linuxdir); 3454 3455 return d_name ? 3456 create_dir_entry(strdup(d_name->d_name), NULL, NULL, dir) : 3457 NULL; 3458 } 3459 3460 3461 void scan1_freedir(struct dir_info *dir) 3462 { 3463 if(dir->pathname[0] != '\0') 3464 closedir(dir->linuxdir); 3465 } 3466 3467 3468 struct dir_info *dir_scan1(char *filename, char *subpath, 3469 struct pathnames *paths, 3470 struct dir_ent *(_readdir)(struct dir_info *), int depth) 3471 { 3472 struct dir_info *dir = scan1_opendir(filename, subpath, depth); 3473 struct dir_ent *dir_ent; 3474 3475 if(dir == NULL) { 3476 ERROR_START("Could not open %s", filename); 3477 ERROR_EXIT(", skipping...\n"); 3478 return NULL; 3479 } 3480 3481 while((dir_ent = _readdir(dir))) { 3482 struct dir_info *sub_dir; 3483 struct stat buf; 3484 struct pathnames *new = NULL; 3485 char *filename = pathname(dir_ent); 3486 char *subpath = NULL; 3487 char *dir_name = dir_ent->name; 3488 3489 if(strcmp(dir_name, ".") == 0 || strcmp(dir_name, "..") == 0) { 3490 free_dir_entry(dir_ent); 3491 continue; 3492 } 3493 3494 if(lstat(filename, &buf) == -1) { 3495 ERROR_START("Cannot stat dir/file %s because %s", 3496 filename, strerror(errno)); 3497 ERROR_EXIT(", ignoring\n"); 3498 free_dir_entry(dir_ent); 3499 continue; 3500 } 3501 3502 if((buf.st_mode & S_IFMT) != S_IFREG && 3503 (buf.st_mode & S_IFMT) != S_IFDIR && 3504 (buf.st_mode & S_IFMT) != S_IFLNK && 3505 (buf.st_mode & S_IFMT) != S_IFCHR && 3506 (buf.st_mode & S_IFMT) != S_IFBLK && 3507 (buf.st_mode & S_IFMT) != S_IFIFO && 3508 (buf.st_mode & S_IFMT) != S_IFSOCK) { 3509 ERROR_START("File %s has unrecognised filetype %d", 3510 filename, buf.st_mode & S_IFMT); 3511 ERROR_EXIT(", ignoring\n"); 3512 free_dir_entry(dir_ent); 3513 continue; 3514 } 3515 3516 if((old_exclude && old_excluded(filename, &buf)) || 3517 (!old_exclude && excluded(dir_name, paths, &new))) { 3518 add_excluded(dir); 3519 free_dir_entry(dir_ent); 3520 continue; 3521 } 3522 3523 if(exclude_actions()) { 3524 subpath = subpathname(dir_ent); 3525 3526 if(eval_exclude_actions(dir_name, filename, subpath, 3527 &buf, depth, dir_ent)) { 3528 add_excluded(dir); 3529 free_dir_entry(dir_ent); 3530 continue; 3531 } 3532 } 3533 3534 switch(buf.st_mode & S_IFMT) { 3535 case S_IFDIR: 3536 if(subpath == NULL) 3537 subpath = subpathname(dir_ent); 3538 3539 sub_dir = dir_scan1(filename, subpath, new, 3540 scan1_readdir, depth + 1); 3541 if(sub_dir) { 3542 dir->directory_count ++; 3543 add_dir_entry(dir_ent, sub_dir, 3544 lookup_inode(&buf)); 3545 } else 3546 free_dir_entry(dir_ent); 3547 break; 3548 case S_IFLNK: { 3549 int byte; 3550 static char buff[65536]; /* overflow safe */ 3551 3552 byte = readlink(filename, buff, 65536); 3553 if(byte == -1) { 3554 ERROR_START("Failed to read symlink %s", 3555 filename); 3556 ERROR_EXIT(", ignoring\n"); 3557 } else if(byte == 65536) { 3558 ERROR_START("Symlink %s is greater than 65536 " 3559 "bytes!", filename); 3560 ERROR_EXIT(", ignoring\n"); 3561 } else { 3562 /* readlink doesn't 0 terminate the returned 3563 * path */ 3564 buff[byte] = '\0'; 3565 add_dir_entry(dir_ent, NULL, lookup_inode3(&buf, 3566 0, 0, buff, byte + 1)); 3567 } 3568 break; 3569 } 3570 default: 3571 add_dir_entry(dir_ent, NULL, lookup_inode(&buf)); 3572 } 3573 3574 free(new); 3575 } 3576 3577 scan1_freedir(dir); 3578 3579 return dir; 3580 } 3581 3582 3583 /* 3584 * dir_scan2 routines... 3585 * This processes most actions and any pseudo files 3586 */ 3587 struct dir_ent *scan2_readdir(struct dir_info *dir, struct dir_ent *dir_ent) 3588 { 3589 if (dir_ent == NULL) 3590 dir_ent = dir->list; 3591 else 3592 dir_ent = dir_ent->next; 3593 3594 for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next); 3595 3596 return dir_ent; 3597 } 3598 3599 3600 struct dir_ent *scan2_lookup(struct dir_info *dir, char *name) 3601 { 3602 struct dir_ent *dir_ent = dir->list; 3603 3604 for(; dir_ent && strcmp(dir_ent->name, name) != 0; 3605 dir_ent = dir_ent->next); 3606 3607 return dir_ent; 3608 } 3609 3610 3611 void dir_scan2(struct dir_info *dir, struct pseudo *pseudo) 3612 { 3613 struct dir_ent *dir_ent = NULL; 3614 struct pseudo_entry *pseudo_ent; 3615 struct stat buf; 3616 static int pseudo_ino = 1; 3617 3618 while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) { 3619 struct inode_info *inode_info = dir_ent->inode; 3620 struct stat *buf = &inode_info->buf; 3621 char *name = dir_ent->name; 3622 3623 eval_actions(root_dir, dir_ent); 3624 3625 if((buf->st_mode & S_IFMT) == S_IFDIR) 3626 dir_scan2(dir_ent->dir, pseudo_subdir(name, pseudo)); 3627 } 3628 3629 while((pseudo_ent = pseudo_readdir(pseudo)) != NULL) { 3630 dir_ent = scan2_lookup(dir, pseudo_ent->name); 3631 if(pseudo_ent->dev->type == 'm') { 3632 struct stat *buf; 3633 if(dir_ent == NULL) { 3634 ERROR_START("Pseudo modify file \"%s\" does " 3635 "not exist in source filesystem.", 3636 pseudo_ent->pathname); 3637 ERROR_EXIT(" Ignoring.\n"); 3638 continue; 3639 } 3640 if(dir_ent->inode->root_entry) { 3641 ERROR_START("Pseudo modify file \"%s\" is a " 3642 "pre-existing file in the filesystem " 3643 "being appended to. It cannot be "\ 3644 "modified.", pseudo_ent->pathname); 3645 ERROR_EXIT(" Ignoring.\n"); 3646 continue; 3647 } 3648 buf = &dir_ent->inode->buf; 3649 buf->st_mode = (buf->st_mode & S_IFMT) | 3650 pseudo_ent->dev->mode; 3651 buf->st_uid = pseudo_ent->dev->uid; 3652 buf->st_gid = pseudo_ent->dev->gid; 3653 continue; 3654 } 3655 3656 if(dir_ent) { 3657 if(dir_ent->inode->root_entry) { 3658 ERROR_START("Pseudo file \"%s\" is a " 3659 "pre-existing file in the filesystem " 3660 "being appended to.", 3661 pseudo_ent->pathname); 3662 ERROR_EXIT(" Ignoring.\n"); 3663 } else { 3664 ERROR_START("Pseudo file \"%s\" exists in " 3665 "source filesystem \"%s\".", 3666 pseudo_ent->pathname, 3667 pathname(dir_ent)); 3668 ERROR_EXIT("\nIgnoring, exclude it (-e/-ef) to " 3669 "override.\n"); 3670 } 3671 continue; 3672 } 3673 3674 memset(&buf, 0, sizeof(buf)); 3675 buf.st_mode = pseudo_ent->dev->mode; 3676 buf.st_uid = pseudo_ent->dev->uid; 3677 buf.st_gid = pseudo_ent->dev->gid; 3678 buf.st_rdev = makedev(pseudo_ent->dev->major, 3679 pseudo_ent->dev->minor); 3680 buf.st_mtime = time(NULL); 3681 buf.st_ino = pseudo_ino ++; 3682 3683 if(pseudo_ent->dev->type == 'd') { 3684 struct dir_ent *dir_ent = 3685 create_dir_entry(pseudo_ent->name, NULL, 3686 pseudo_ent->pathname, dir); 3687 char *subpath = strdup(subpathname(dir_ent)); 3688 struct dir_info *sub_dir = scan1_opendir("", subpath, 3689 dir->depth + 1); 3690 if(sub_dir == NULL) { 3691 ERROR_START("Could not create pseudo directory " 3692 "\"%s\"", pseudo_ent->pathname); 3693 ERROR_EXIT(", skipping...\n"); 3694 free(subpath); 3695 pseudo_ino --; 3696 continue; 3697 } 3698 dir_scan2(sub_dir, pseudo_ent->pseudo); 3699 dir->directory_count ++; 3700 add_dir_entry(dir_ent, sub_dir, 3701 lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0)); 3702 } else if(pseudo_ent->dev->type == 'f') { 3703 add_dir_entry2(pseudo_ent->name, NULL, 3704 pseudo_ent->pathname, NULL, 3705 lookup_inode2(&buf, PSEUDO_FILE_PROCESS, 3706 pseudo_ent->dev->pseudo_id), dir); 3707 } else { 3708 add_dir_entry2(pseudo_ent->name, NULL, 3709 pseudo_ent->pathname, NULL, 3710 lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0), dir); 3711 } 3712 } 3713 } 3714 3715 3716 /* 3717 * dir_scan3 routines... 3718 * This processes the move action 3719 */ 3720 void dir_scan3(struct dir_info *dir) 3721 { 3722 struct dir_ent *dir_ent = NULL; 3723 3724 while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) { 3725 3726 eval_move_actions(root_dir, dir_ent); 3727 3728 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3729 dir_scan3(dir_ent->dir); 3730 } 3731 } 3732 3733 3734 /* 3735 * dir_scan4 routines... 3736 * This processes the prune action. This action is designed to do fine 3737 * grained tuning of the in-core directory structure after the exclude, 3738 * move and pseudo actions have been performed. This allows complex 3739 * tests to be performed which are impossible at exclude time (i.e. 3740 * tests which rely on the in-core directory structure) 3741 */ 3742 void free_dir(struct dir_info *dir) 3743 { 3744 struct dir_ent *dir_ent = dir->list; 3745 3746 while(dir_ent) { 3747 struct dir_ent *tmp = dir_ent; 3748 3749 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3750 free_dir(dir_ent->dir); 3751 3752 dir_ent = dir_ent->next; 3753 free_dir_entry(tmp); 3754 } 3755 3756 free(dir->pathname); 3757 free(dir->subpath); 3758 free(dir); 3759 } 3760 3761 3762 void dir_scan4(struct dir_info *dir) 3763 { 3764 struct dir_ent *dir_ent = dir->list, *prev = NULL; 3765 3766 while(dir_ent) { 3767 if(dir_ent->inode->root_entry) { 3768 prev = dir_ent; 3769 dir_ent = dir_ent->next; 3770 continue; 3771 } 3772 3773 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3774 dir_scan4(dir_ent->dir); 3775 3776 if(eval_prune_actions(root_dir, dir_ent)) { 3777 struct dir_ent *tmp = dir_ent; 3778 3779 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { 3780 free_dir(dir_ent->dir); 3781 dir->directory_count --; 3782 } 3783 3784 dir->count --; 3785 3786 /* remove dir_ent from list */ 3787 dir_ent = dir_ent->next; 3788 if(prev) 3789 prev->next = dir_ent; 3790 else 3791 dir->list = dir_ent; 3792 3793 /* free it */ 3794 free_dir_entry(tmp); 3795 3796 add_excluded(dir); 3797 continue; 3798 } 3799 3800 prev = dir_ent; 3801 dir_ent = dir_ent->next; 3802 } 3803 } 3804 3805 3806 /* 3807 * dir_scan5 routines... 3808 * This processes the empty action. This action has to be processed after 3809 * all other actions because the previous exclude and move actions and the 3810 * pseudo actions affect whether a directory is empty 3811 */ 3812 void dir_scan5(struct dir_info *dir) 3813 { 3814 struct dir_ent *dir_ent = dir->list, *prev = NULL; 3815 3816 while(dir_ent) { 3817 if(dir_ent->inode->root_entry) { 3818 prev = dir_ent; 3819 dir_ent = dir_ent->next; 3820 continue; 3821 } 3822 3823 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { 3824 dir_scan5(dir_ent->dir); 3825 3826 if(eval_empty_actions(root_dir, dir_ent)) { 3827 struct dir_ent *tmp = dir_ent; 3828 3829 /* 3830 * delete sub-directory, this is by definition 3831 * empty 3832 */ 3833 free(dir_ent->dir->pathname); 3834 free(dir_ent->dir->subpath); 3835 free(dir_ent->dir); 3836 3837 /* remove dir_ent from list */ 3838 dir_ent = dir_ent->next; 3839 if(prev) 3840 prev->next = dir_ent; 3841 else 3842 dir->list = dir_ent; 3843 3844 /* free it */ 3845 free_dir_entry(tmp); 3846 3847 /* update counts */ 3848 dir->directory_count --; 3849 dir->count --; 3850 add_excluded(dir); 3851 continue; 3852 } 3853 } 3854 3855 prev = dir_ent; 3856 dir_ent = dir_ent->next; 3857 } 3858 } 3859 3860 3861 /* 3862 * dir_scan6 routines... 3863 * This sorts every directory and computes the inode numbers 3864 */ 3865 3866 /* 3867 * Bottom up linked list merge sort. 3868 * 3869 * Qsort and other O(n log n) algorithms work well with arrays but not 3870 * linked lists. Merge sort another O(n log n) sort algorithm on the other hand 3871 * is not ideal for arrays (as it needs an additonal n storage locations 3872 * as sorting is not done in place), but it is ideal for linked lists because 3873 * it doesn't require any extra storage, 3874 */ 3875 void sort_directory(struct dir_info *dir) 3876 { 3877 struct dir_ent *cur, *l1, *l2, *next; 3878 int len1, len2, stride = 1; 3879 3880 if(dir->list == NULL || dir->count < 2) 3881 return; 3882 3883 /* 3884 * We can consider our linked-list to be made up of stride length 3885 * sublists. Eacn iteration around this loop merges adjacent 3886 * stride length sublists into larger 2*stride sublists. We stop 3887 * when stride becomes equal to the entire list. 3888 * 3889 * Initially stride = 1 (by definition a sublist of 1 is sorted), and 3890 * these 1 element sublists are merged into 2 element sublists, which 3891 * are then merged into 4 element sublists and so on. 3892 */ 3893 do { 3894 l2 = dir->list; /* head of current linked list */ 3895 cur = NULL; /* empty output list */ 3896 3897 /* 3898 * Iterate through the linked list, merging adjacent sublists. 3899 * On each interation l2 points to the next sublist pair to be 3900 * merged (if there's only one sublist left this is simply added 3901 * to the output list) 3902 */ 3903 while(l2) { 3904 l1 = l2; 3905 for(len1 = 0; l2 && len1 < stride; len1 ++, l2 = l2->next); 3906 len2 = stride; 3907 3908 /* 3909 * l1 points to first sublist. 3910 * l2 points to second sublist. 3911 * Merge them onto the output list 3912 */ 3913 while(len1 && l2 && len2) { 3914 if(strcmp(l1->name, l2->name) <= 0) { 3915 next = l1; 3916 l1 = l1->next; 3917 len1 --; 3918 } else { 3919 next = l2; 3920 l2 = l2->next; 3921 len2 --; 3922 } 3923 3924 if(cur) { 3925 cur->next = next; 3926 cur = next; 3927 } else 3928 dir->list = cur = next; 3929 } 3930 /* 3931 * One sublist is now empty, copy the other one onto the 3932 * output list 3933 */ 3934 for(; len1; len1 --, l1 = l1->next) { 3935 if(cur) { 3936 cur->next = l1; 3937 cur = l1; 3938 } else 3939 dir->list = cur = l1; 3940 } 3941 for(; l2 && len2; len2 --, l2 = l2->next) { 3942 if(cur) { 3943 cur->next = l2; 3944 cur = l2; 3945 } else 3946 dir->list = cur = l2; 3947 } 3948 } 3949 cur->next = NULL; 3950 stride = stride << 1; 3951 } while(stride < dir->count); 3952 } 3953 3954 3955 void dir_scan6(struct dir_info *dir) 3956 { 3957 struct dir_ent *dir_ent; 3958 unsigned int byte_count = 0; 3959 3960 sort_directory(dir); 3961 3962 for(dir_ent = dir->list; dir_ent; dir_ent = dir_ent->next) { 3963 byte_count += strlen(dir_ent->name) + 3964 sizeof(struct squashfs_dir_entry); 3965 3966 if(dir_ent->inode->root_entry) 3967 continue; 3968 3969 alloc_inode_no(dir_ent->inode, 0); 3970 3971 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3972 dir_scan6(dir_ent->dir); 3973 } 3974 3975 if((dir->count < 257 && byte_count < SQUASHFS_METADATA_SIZE)) 3976 dir->dir_is_ldir = FALSE; 3977 } 3978 3979 3980 /* 3981 * dir_scan6 routines... 3982 * This generates the filesystem metadata and writes it out to the destination 3983 */ 3984 void scan7_init_dir(struct directory *dir) 3985 { 3986 dir->buff = malloc(SQUASHFS_METADATA_SIZE); 3987 if(dir->buff == NULL) 3988 MEM_ERROR(); 3989 3990 dir->size = SQUASHFS_METADATA_SIZE; 3991 dir->p = dir->index_count_p = dir->buff; 3992 dir->entry_count = 256; 3993 dir->entry_count_p = NULL; 3994 dir->index = NULL; 3995 dir->i_count = dir->i_size = 0; 3996 } 3997 3998 3999 struct dir_ent *scan7_readdir(struct directory *dir, struct dir_info *dir_info, 4000 struct dir_ent *dir_ent) 4001 { 4002 if (dir_ent == NULL) 4003 dir_ent = dir_info->list; 4004 else 4005 dir_ent = dir_ent->next; 4006 4007 for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next) 4008 add_dir(dir_ent->inode->inode, dir_ent->inode->inode_number, 4009 dir_ent->name, dir_ent->inode->type, dir); 4010 4011 return dir_ent; 4012 } 4013 4014 4015 void scan7_freedir(struct directory *dir) 4016 { 4017 if(dir->index) 4018 free(dir->index); 4019 free(dir->buff); 4020 } 4021 4022 4023 void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info) 4024 { 4025 int squashfs_type; 4026 int duplicate_file; 4027 struct directory dir; 4028 struct dir_ent *dir_ent = NULL; 4029 4030 scan7_init_dir(&dir); 4031 4032 while((dir_ent = scan7_readdir(&dir, dir_info, dir_ent)) != NULL) { 4033 struct stat *buf = &dir_ent->inode->buf; 4034 4035 update_info(dir_ent); 4036 4037 if(dir_ent->inode->inode == SQUASHFS_INVALID_BLK) { 4038 switch(buf->st_mode & S_IFMT) { 4039 case S_IFREG: 4040 squashfs_type = SQUASHFS_FILE_TYPE; 4041 write_file(inode, dir_ent, 4042 &duplicate_file); 4043 INFO("file %s, uncompressed size %lld " 4044 "bytes %s\n", 4045 subpathname(dir_ent), 4046 (long long) buf->st_size, 4047 duplicate_file ? "DUPLICATE" : 4048 ""); 4049 break; 4050 4051 case S_IFDIR: 4052 squashfs_type = SQUASHFS_DIR_TYPE; 4053 dir_scan7(inode, dir_ent->dir); 4054 break; 4055 4056 case S_IFLNK: 4057 squashfs_type = SQUASHFS_SYMLINK_TYPE; 4058 create_inode(inode, NULL, dir_ent, 4059 squashfs_type, 0, 0, 0, NULL, 4060 NULL, NULL, 0); 4061 INFO("symbolic link %s inode 0x%llx\n", 4062 subpathname(dir_ent), *inode); 4063 sym_count ++; 4064 break; 4065 4066 case S_IFCHR: 4067 squashfs_type = SQUASHFS_CHRDEV_TYPE; 4068 create_inode(inode, NULL, dir_ent, 4069 squashfs_type, 0, 0, 0, NULL, 4070 NULL, NULL, 0); 4071 INFO("character device %s inode 0x%llx" 4072 "\n", subpathname(dir_ent), 4073 *inode); 4074 dev_count ++; 4075 break; 4076 4077 case S_IFBLK: 4078 squashfs_type = SQUASHFS_BLKDEV_TYPE; 4079 create_inode(inode, NULL, dir_ent, 4080 squashfs_type, 0, 0, 0, NULL, 4081 NULL, NULL, 0); 4082 INFO("block device %s inode 0x%llx\n", 4083 subpathname(dir_ent), *inode); 4084 dev_count ++; 4085 break; 4086 4087 case S_IFIFO: 4088 squashfs_type = SQUASHFS_FIFO_TYPE; 4089 create_inode(inode, NULL, dir_ent, 4090 squashfs_type, 0, 0, 0, NULL, 4091 NULL, NULL, 0); 4092 INFO("fifo %s inode 0x%llx\n", 4093 subpathname(dir_ent), *inode); 4094 fifo_count ++; 4095 break; 4096 4097 case S_IFSOCK: 4098 squashfs_type = SQUASHFS_SOCKET_TYPE; 4099 create_inode(inode, NULL, dir_ent, 4100 squashfs_type, 0, 0, 0, NULL, 4101 NULL, NULL, 0); 4102 INFO("unix domain socket %s inode " 4103 "0x%llx\n", 4104 subpathname(dir_ent), *inode); 4105 sock_count ++; 4106 break; 4107 4108 default: 4109 BAD_ERROR("%s unrecognised file type, " 4110 "mode is %x\n", 4111 subpathname(dir_ent), 4112 buf->st_mode); 4113 } 4114 dir_ent->inode->inode = *inode; 4115 dir_ent->inode->type = squashfs_type; 4116 } else { 4117 *inode = dir_ent->inode->inode; 4118 squashfs_type = dir_ent->inode->type; 4119 switch(squashfs_type) { 4120 case SQUASHFS_FILE_TYPE: 4121 if(!sorted) 4122 INFO("file %s, uncompressed " 4123 "size %lld bytes LINK" 4124 "\n", 4125 subpathname(dir_ent), 4126 (long long) 4127 buf->st_size); 4128 break; 4129 case SQUASHFS_SYMLINK_TYPE: 4130 INFO("symbolic link %s inode 0x%llx " 4131 "LINK\n", subpathname(dir_ent), 4132 *inode); 4133 break; 4134 case SQUASHFS_CHRDEV_TYPE: 4135 INFO("character device %s inode 0x%llx " 4136 "LINK\n", subpathname(dir_ent), 4137 *inode); 4138 break; 4139 case SQUASHFS_BLKDEV_TYPE: 4140 INFO("block device %s inode 0x%llx " 4141 "LINK\n", subpathname(dir_ent), 4142 *inode); 4143 break; 4144 case SQUASHFS_FIFO_TYPE: 4145 INFO("fifo %s inode 0x%llx LINK\n", 4146 subpathname(dir_ent), *inode); 4147 break; 4148 case SQUASHFS_SOCKET_TYPE: 4149 INFO("unix domain socket %s inode " 4150 "0x%llx LINK\n", 4151 subpathname(dir_ent), *inode); 4152 break; 4153 } 4154 } 4155 4156 add_dir(*inode, get_inode_no(dir_ent->inode), dir_ent->name, 4157 squashfs_type, &dir); 4158 } 4159 4160 write_dir(inode, dir_info, &dir); 4161 INFO("directory %s inode 0x%llx\n", subpathname(dir_info->dir_ent), 4162 *inode); 4163 4164 scan7_freedir(&dir); 4165 } 4166 4167 4168 unsigned int slog(unsigned int block) 4169 { 4170 int i; 4171 4172 for(i = 12; i <= 20; i++) 4173 if(block == (1 << i)) 4174 return i; 4175 return 0; 4176 } 4177 4178 4179 int old_excluded(char *filename, struct stat *buf) 4180 { 4181 int i; 4182 4183 for(i = 0; i < exclude; i++) 4184 if((exclude_paths[i].st_dev == buf->st_dev) && 4185 (exclude_paths[i].st_ino == buf->st_ino)) 4186 return TRUE; 4187 return FALSE; 4188 } 4189 4190 4191 #define ADD_ENTRY(buf) \ 4192 if(exclude % EXCLUDE_SIZE == 0) { \ 4193 exclude_paths = realloc(exclude_paths, (exclude + EXCLUDE_SIZE) \ 4194 * sizeof(struct exclude_info)); \ 4195 if(exclude_paths == NULL) \ 4196 MEM_ERROR(); \ 4197 } \ 4198 exclude_paths[exclude].st_dev = buf.st_dev; \ 4199 exclude_paths[exclude++].st_ino = buf.st_ino; 4200 int old_add_exclude(char *path) 4201 { 4202 int i; 4203 char *filename; 4204 struct stat buf; 4205 4206 if(path[0] == '/' || strncmp(path, "./", 2) == 0 || 4207 strncmp(path, "../", 3) == 0) { 4208 if(lstat(path, &buf) == -1) { 4209 ERROR_START("Cannot stat exclude dir/file %s because " 4210 "%s", path, strerror(errno)); 4211 ERROR_EXIT(", ignoring\n"); 4212 return TRUE; 4213 } 4214 ADD_ENTRY(buf); 4215 return TRUE; 4216 } 4217 4218 for(i = 0; i < source; i++) { 4219 int res = asprintf(&filename, "%s/%s", source_path[i], path); 4220 if(res == -1) 4221 BAD_ERROR("asprintf failed in old_add_exclude\n"); 4222 if(lstat(filename, &buf) == -1) { 4223 if(!(errno == ENOENT || errno == ENOTDIR)) { 4224 ERROR_START("Cannot stat exclude dir/file %s " 4225 "because %s", filename, strerror(errno)); 4226 ERROR_EXIT(", ignoring\n"); 4227 } 4228 free(filename); 4229 continue; 4230 } 4231 free(filename); 4232 ADD_ENTRY(buf); 4233 } 4234 return TRUE; 4235 } 4236 4237 4238 void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, 4239 int type) 4240 { 4241 old_root_entry = realloc(old_root_entry, 4242 sizeof(struct old_root_entry_info) * (old_root_entries + 1)); 4243 if(old_root_entry == NULL) 4244 MEM_ERROR(); 4245 4246 old_root_entry[old_root_entries].name = strdup(name); 4247 old_root_entry[old_root_entries].inode.inode = inode; 4248 old_root_entry[old_root_entries].inode.inode_number = inode_number; 4249 old_root_entry[old_root_entries].inode.type = type; 4250 old_root_entry[old_root_entries++].inode.root_entry = TRUE; 4251 } 4252 4253 4254 void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, 4255 int freelst, char *destination_file) 4256 { 4257 int i; 4258 sigset_t sigmask, old_mask; 4259 int total_mem = readq; 4260 int reader_size; 4261 int fragment_size; 4262 int fwriter_size; 4263 /* 4264 * bwriter_size is global because it is needed in 4265 * write_file_blocks_dup() 4266 */ 4267 4268 /* 4269 * Never allow the total size of the queues to be larger than 4270 * physical memory 4271 * 4272 * When adding together the possibly user supplied values, make 4273 * sure they've not been deliberately contrived to overflow an int 4274 */ 4275 if(add_overflow(total_mem, fragq)) 4276 BAD_ERROR("Queue sizes rediculously too large\n"); 4277 total_mem += fragq; 4278 if(add_overflow(total_mem, bwriteq)) 4279 BAD_ERROR("Queue sizes rediculously too large\n"); 4280 total_mem += bwriteq; 4281 if(add_overflow(total_mem, fwriteq)) 4282 BAD_ERROR("Queue sizes rediculously too large\n"); 4283 total_mem += fwriteq; 4284 4285 check_usable_phys_mem(total_mem); 4286 4287 /* 4288 * convert from queue size in Mbytes to queue size in 4289 * blocks. 4290 * 4291 * This isn't going to overflow an int unless there exists 4292 * systems with more than 8 Petabytes of RAM! 4293 */ 4294 reader_size = readq << (20 - block_log); 4295 fragment_size = fragq << (20 - block_log); 4296 bwriter_size = bwriteq << (20 - block_log); 4297 fwriter_size = fwriteq << (20 - block_log); 4298 4299 /* 4300 * setup signal handlers for the main thread, these cleanup 4301 * deleting the destination file, if appending the 4302 * handlers for SIGTERM and SIGINT will be replaced with handlers 4303 * allowing the user to press ^C twice to restore the existing 4304 * filesystem. 4305 * 4306 * SIGUSR1 is an internal signal, which is used by the sub-threads 4307 * to tell the main thread to terminate, deleting the destination file, 4308 * or if necessary restoring the filesystem on appending 4309 */ 4310 signal(SIGTERM, sighandler); 4311 signal(SIGINT, sighandler); 4312 signal(SIGUSR1, sighandler); 4313 4314 /* block SIGQUIT and SIGHUP, these are handled by the info thread */ 4315 sigemptyset(&sigmask); 4316 sigaddset(&sigmask, SIGQUIT); 4317 sigaddset(&sigmask, SIGHUP); 4318 sigaddset(&sigmask, SIGALRM); 4319 if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1) 4320 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4321 4322 /* 4323 * temporarily block these signals, so the created sub-threads 4324 * will ignore them, ensuring the main thread handles them 4325 */ 4326 sigemptyset(&sigmask); 4327 sigaddset(&sigmask, SIGINT); 4328 sigaddset(&sigmask, SIGTERM); 4329 sigaddset(&sigmask, SIGUSR1); 4330 if(pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask) == -1) 4331 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4332 4333 if(processors == -1) { 4334 #ifndef linux 4335 int mib[2]; 4336 size_t len = sizeof(processors); 4337 4338 mib[0] = CTL_HW; 4339 #ifdef HW_AVAILCPU 4340 mib[1] = HW_AVAILCPU; 4341 #else 4342 mib[1] = HW_NCPU; 4343 #endif 4344 4345 if(sysctl(mib, 2, &processors, &len, NULL, 0) == -1) { 4346 ERROR_START("Failed to get number of available " 4347 "processors."); 4348 ERROR_EXIT(" Defaulting to 1\n"); 4349 processors = 1; 4350 } 4351 #else 4352 processors = sysconf(_SC_NPROCESSORS_ONLN); 4353 #endif 4354 } 4355 4356 if(multiply_overflow(processors, 3) || 4357 multiply_overflow(processors * 3, sizeof(pthread_t))) 4358 BAD_ERROR("Processors too large\n"); 4359 4360 deflator_thread = malloc(processors * 3 * sizeof(pthread_t)); 4361 if(deflator_thread == NULL) 4362 MEM_ERROR(); 4363 4364 frag_deflator_thread = &deflator_thread[processors]; 4365 frag_thread = &frag_deflator_thread[processors]; 4366 4367 to_reader = queue_init(1); 4368 to_deflate = queue_init(reader_size); 4369 to_process_frag = queue_init(reader_size); 4370 to_writer = queue_init(bwriter_size + fwriter_size); 4371 from_writer = queue_init(1); 4372 to_frag = queue_init(fragment_size); 4373 locked_fragment = queue_init(fragment_size); 4374 to_main = seq_queue_init(); 4375 reader_buffer = cache_init(block_size, reader_size, 0, 0); 4376 bwriter_buffer = cache_init(block_size, bwriter_size, 1, freelst); 4377 fwriter_buffer = cache_init(block_size, fwriter_size, 1, freelst); 4378 fragment_buffer = cache_init(block_size, fragment_size, 1, 0); 4379 reserve_cache = cache_init(block_size, processors + 1, 1, 0); 4380 pthread_create(&reader_thread, NULL, reader, NULL); 4381 pthread_create(&writer_thread, NULL, writer, NULL); 4382 init_progress_bar(); 4383 init_info(); 4384 4385 for(i = 0; i < processors; i++) { 4386 if(pthread_create(&deflator_thread[i], NULL, deflator, NULL)) 4387 BAD_ERROR("Failed to create thread\n"); 4388 if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator, 4389 NULL) != 0) 4390 BAD_ERROR("Failed to create thread\n"); 4391 if(pthread_create(&frag_thread[i], NULL, frag_thrd, 4392 (void *) destination_file) != 0) 4393 BAD_ERROR("Failed to create thread\n"); 4394 } 4395 4396 main_thread = pthread_self(); 4397 4398 printf("Parallel mksquashfs: Using %d processor%s\n", processors, 4399 processors == 1 ? "" : "s"); 4400 4401 /* Restore the signal mask for the main thread */ 4402 if(pthread_sigmask(SIG_SETMASK, &old_mask, NULL) == -1) 4403 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4404 } 4405 4406 4407 long long write_inode_lookup_table() 4408 { 4409 int i, inode_number, lookup_bytes = SQUASHFS_LOOKUP_BYTES(inode_count); 4410 void *it; 4411 4412 if(inode_count == sinode_count) 4413 goto skip_inode_hash_table; 4414 4415 it = realloc(inode_lookup_table, lookup_bytes); 4416 if(it == NULL) 4417 MEM_ERROR(); 4418 inode_lookup_table = it; 4419 4420 for(i = 0; i < INODE_HASH_SIZE; i ++) { 4421 struct inode_info *inode; 4422 4423 for(inode = inode_info[i]; inode; inode = inode->next) { 4424 4425 inode_number = get_inode_no(inode); 4426 4427 /* The empty action will produce orphaned inode 4428 * entries in the inode_info[] table. These 4429 * entries because they are orphaned will not be 4430 * allocated an inode number in dir_scan5(), so 4431 * skip any entries with the default dummy inode 4432 * number of 0 */ 4433 if(inode_number == 0) 4434 continue; 4435 4436 SQUASHFS_SWAP_LONG_LONGS(&inode->inode, 4437 &inode_lookup_table[inode_number - 1], 1); 4438 4439 } 4440 } 4441 4442 skip_inode_hash_table: 4443 return generic_write_table(lookup_bytes, inode_lookup_table, 0, NULL, 4444 noI); 4445 } 4446 4447 4448 char *get_component(char *target, char **targname) 4449 { 4450 char *start; 4451 4452 while(*target == '/') 4453 target ++; 4454 4455 start = target; 4456 while(*target != '/' && *target != '\0') 4457 target ++; 4458 4459 *targname = strndup(start, target - start); 4460 4461 while(*target == '/') 4462 target ++; 4463 4464 return target; 4465 } 4466 4467 4468 void free_path(struct pathname *paths) 4469 { 4470 int i; 4471 4472 for(i = 0; i < paths->names; i++) { 4473 if(paths->name[i].paths) 4474 free_path(paths->name[i].paths); 4475 free(paths->name[i].name); 4476 if(paths->name[i].preg) { 4477 regfree(paths->name[i].preg); 4478 free(paths->name[i].preg); 4479 } 4480 } 4481 4482 free(paths); 4483 } 4484 4485 4486 struct pathname *add_path(struct pathname *paths, char *target, char *alltarget) 4487 { 4488 char *targname; 4489 int i, error; 4490 4491 target = get_component(target, &targname); 4492 4493 if(paths == NULL) { 4494 paths = malloc(sizeof(struct pathname)); 4495 if(paths == NULL) 4496 MEM_ERROR(); 4497 4498 paths->names = 0; 4499 paths->name = NULL; 4500 } 4501 4502 for(i = 0; i < paths->names; i++) 4503 if(strcmp(paths->name[i].name, targname) == 0) 4504 break; 4505 4506 if(i == paths->names) { 4507 /* allocate new name entry */ 4508 paths->names ++; 4509 paths->name = realloc(paths->name, (i + 1) * 4510 sizeof(struct path_entry)); 4511 if(paths->name == NULL) 4512 MEM_ERROR(); 4513 paths->name[i].name = targname; 4514 paths->name[i].paths = NULL; 4515 if(use_regex) { 4516 paths->name[i].preg = malloc(sizeof(regex_t)); 4517 if(paths->name[i].preg == NULL) 4518 MEM_ERROR(); 4519 error = regcomp(paths->name[i].preg, targname, 4520 REG_EXTENDED|REG_NOSUB); 4521 if(error) { 4522 char str[1024]; /* overflow safe */ 4523 4524 regerror(error, paths->name[i].preg, str, 1024); 4525 BAD_ERROR("invalid regex %s in export %s, " 4526 "because %s\n", targname, alltarget, 4527 str); 4528 } 4529 } else 4530 paths->name[i].preg = NULL; 4531 4532 if(target[0] == '\0') 4533 /* at leaf pathname component */ 4534 paths->name[i].paths = NULL; 4535 else 4536 /* recurse adding child components */ 4537 paths->name[i].paths = add_path(NULL, target, 4538 alltarget); 4539 } else { 4540 /* existing matching entry */ 4541 free(targname); 4542 4543 if(paths->name[i].paths == NULL) { 4544 /* No sub-directory which means this is the leaf 4545 * component of a pre-existing exclude which subsumes 4546 * the exclude currently being added, in which case stop 4547 * adding components */ 4548 } else if(target[0] == '\0') { 4549 /* at leaf pathname component and child components exist 4550 * from more specific excludes, delete as they're 4551 * subsumed by this exclude */ 4552 free_path(paths->name[i].paths); 4553 paths->name[i].paths = NULL; 4554 } else 4555 /* recurse adding child components */ 4556 add_path(paths->name[i].paths, target, alltarget); 4557 } 4558 4559 return paths; 4560 } 4561 4562 4563 void add_exclude(char *target) 4564 { 4565 4566 if(target[0] == '/' || strncmp(target, "./", 2) == 0 || 4567 strncmp(target, "../", 3) == 0) 4568 BAD_ERROR("/, ./ and ../ prefixed excludes not supported with " 4569 "-wildcards or -regex options\n"); 4570 else if(strncmp(target, "... ", 4) == 0) 4571 stickypath = add_path(stickypath, target + 4, target + 4); 4572 else 4573 path = add_path(path, target, target); 4574 } 4575 4576 4577 void display_path(int depth, struct pathname *paths) 4578 { 4579 int i, n; 4580 4581 if(paths == NULL) 4582 return; 4583 4584 for(i = 0; i < paths->names; i++) { 4585 for(n = 0; n < depth; n++) 4586 printf("\t"); 4587 printf("%d: %s\n", depth, paths->name[i].name); 4588 display_path(depth + 1, paths->name[i].paths); 4589 } 4590 } 4591 4592 4593 void display_path2(struct pathname *paths, char *string) 4594 { 4595 int i; 4596 char *path; 4597 4598 if(paths == NULL) { 4599 printf("%s\n", string); 4600 return; 4601 } 4602 4603 for(i = 0; i < paths->names; i++) { 4604 int res = asprintf(&path, "%s/%s", string, paths->name[i].name); 4605 if(res == -1) 4606 BAD_ERROR("asprintf failed in display_path2\n"); 4607 display_path2(paths->name[i].paths, path); 4608 free(path); 4609 } 4610 } 4611 4612 4613 struct pathnames *add_subdir(struct pathnames *paths, struct pathname *path) 4614 { 4615 int count = paths == NULL ? 0 : paths->count; 4616 4617 if(count % PATHS_ALLOC_SIZE == 0) { 4618 paths = realloc(paths, sizeof(struct pathnames) + 4619 (count + PATHS_ALLOC_SIZE) * sizeof(struct pathname *)); 4620 if(paths == NULL) 4621 MEM_ERROR(); 4622 } 4623 4624 paths->path[count] = path; 4625 paths->count = count + 1; 4626 return paths; 4627 } 4628 4629 4630 int excluded_match(char *name, struct pathname *path, struct pathnames **new) 4631 { 4632 int i; 4633 4634 for(i = 0; i < path->names; i++) { 4635 int match = use_regex ? 4636 regexec(path->name[i].preg, name, (size_t) 0, 4637 NULL, 0) == 0 : 4638 fnmatch(path->name[i].name, name, 4639 FNM_PATHNAME|FNM_PERIOD|FNM_EXTMATCH) == 0; 4640 4641 if(match) { 4642 if(path->name[i].paths == NULL || new == NULL) 4643 /* match on a leaf component, any subdirectories 4644 * in the filesystem should be excluded */ 4645 return TRUE; 4646 else 4647 /* match on a non-leaf component, add any 4648 * subdirectories to the new set of 4649 * subdirectories to scan for this name */ 4650 *new = add_subdir(*new, path->name[i].paths); 4651 } 4652 } 4653 4654 return FALSE; 4655 } 4656 4657 4658 int excluded(char *name, struct pathnames *paths, struct pathnames **new) 4659 { 4660 int n; 4661 4662 if(stickypath && excluded_match(name, stickypath, NULL)) 4663 return TRUE; 4664 4665 for(n = 0; paths && n < paths->count; n++) { 4666 int res = excluded_match(name, paths->path[n], new); 4667 if(res) { 4668 free(*new); 4669 *new = NULL; 4670 return TRUE; 4671 } 4672 } 4673 4674 /* 4675 * Either: 4676 * - no matching names found, return empty new search set, or 4677 * - one or more matches with sub-directories found (no leaf matches), 4678 * in which case return new search set. 4679 * 4680 * In either case return FALSE as we don't want to exclude this entry 4681 */ 4682 return FALSE; 4683 } 4684 4685 4686 void process_exclude_file(char *argv) 4687 { 4688 FILE *fd; 4689 char buffer[MAX_LINE + 1]; /* overflow safe */ 4690 char *filename; 4691 4692 fd = fopen(argv, "r"); 4693 if(fd == NULL) 4694 BAD_ERROR("Failed to open exclude file \"%s\" because %s\n", 4695 argv, strerror(errno)); 4696 4697 while(fgets(filename = buffer, MAX_LINE + 1, fd) != NULL) { 4698 int len = strlen(filename); 4699 4700 if(len == MAX_LINE && filename[len - 1] != '\n') 4701 /* line too large */ 4702 BAD_ERROR("Line too long when reading " 4703 "exclude file \"%s\", larger than %d " 4704 "bytes\n", argv, MAX_LINE); 4705 4706 /* 4707 * Remove '\n' terminator if it exists (the last line 4708 * in the file may not be '\n' terminated) 4709 */ 4710 if(len && filename[len - 1] == '\n') 4711 filename[len - 1] = '\0'; 4712 4713 /* Skip any leading whitespace */ 4714 while(isspace(*filename)) 4715 filename ++; 4716 4717 /* if comment line, skip */ 4718 if(*filename == '#') 4719 continue; 4720 4721 /* 4722 * check for initial backslash, to accommodate 4723 * filenames with leading space or leading # character 4724 */ 4725 if(*filename == '\\') 4726 filename ++; 4727 4728 /* if line is now empty after skipping characters, skip it */ 4729 if(*filename == '\0') 4730 continue; 4731 4732 if(old_exclude) 4733 old_add_exclude(filename); 4734 else 4735 add_exclude(filename); 4736 } 4737 4738 if(ferror(fd)) 4739 BAD_ERROR("Reading exclude file \"%s\" failed because %s\n", 4740 argv, strerror(errno)); 4741 4742 fclose(fd); 4743 } 4744 4745 4746 #define RECOVER_ID "Squashfs recovery file v1.0\n" 4747 #define RECOVER_ID_SIZE 28 4748 4749 void write_recovery_data(struct squashfs_super_block *sBlk) 4750 { 4751 int res, recoverfd, bytes = sBlk->bytes_used - sBlk->inode_table_start; 4752 pid_t pid = getpid(); 4753 char *metadata; 4754 char header[] = RECOVER_ID; 4755 4756 if(recover == FALSE) { 4757 printf("No recovery data option specified.\n"); 4758 printf("Skipping saving recovery file.\n\n"); 4759 return; 4760 } 4761 4762 metadata = malloc(bytes); 4763 if(metadata == NULL) 4764 MEM_ERROR(); 4765 4766 res = read_fs_bytes(fd, sBlk->inode_table_start, bytes, metadata); 4767 if(res == 0) { 4768 ERROR("Failed to read append filesystem metadata\n"); 4769 BAD_ERROR("Filesystem corrupted?\n"); 4770 } 4771 4772 res = asprintf(&recovery_file, "squashfs_recovery_%s_%d", 4773 getbase(destination_file), pid); 4774 if(res == -1) 4775 MEM_ERROR(); 4776 4777 recoverfd = open(recovery_file, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU); 4778 if(recoverfd == -1) 4779 BAD_ERROR("Failed to create recovery file, because %s. " 4780 "Aborting\n", strerror(errno)); 4781 4782 if(write_bytes(recoverfd, header, RECOVER_ID_SIZE) == -1) 4783 BAD_ERROR("Failed to write recovery file, because %s\n", 4784 strerror(errno)); 4785 4786 if(write_bytes(recoverfd, sBlk, sizeof(struct squashfs_super_block)) == -1) 4787 BAD_ERROR("Failed to write recovery file, because %s\n", 4788 strerror(errno)); 4789 4790 if(write_bytes(recoverfd, metadata, bytes) == -1) 4791 BAD_ERROR("Failed to write recovery file, because %s\n", 4792 strerror(errno)); 4793 4794 close(recoverfd); 4795 free(metadata); 4796 4797 printf("Recovery file \"%s\" written\n", recovery_file); 4798 printf("If Mksquashfs aborts abnormally (i.e. power failure), run\n"); 4799 printf("mksquashfs dummy %s -recover %s\n", destination_file, 4800 recovery_file); 4801 printf("to restore filesystem\n\n"); 4802 } 4803 4804 4805 void read_recovery_data(char *recovery_file, char *destination_file) 4806 { 4807 int fd, recoverfd, bytes; 4808 struct squashfs_super_block orig_sBlk, sBlk; 4809 char *metadata; 4810 int res; 4811 struct stat buf; 4812 char header[] = RECOVER_ID; 4813 char header2[RECOVER_ID_SIZE]; 4814 4815 recoverfd = open(recovery_file, O_RDONLY); 4816 if(recoverfd == -1) 4817 BAD_ERROR("Failed to open recovery file because %s\n", 4818 strerror(errno)); 4819 4820 if(stat(destination_file, &buf) == -1) 4821 BAD_ERROR("Failed to stat destination file, because %s\n", 4822 strerror(errno)); 4823 4824 fd = open(destination_file, O_RDWR); 4825 if(fd == -1) 4826 BAD_ERROR("Failed to open destination file because %s\n", 4827 strerror(errno)); 4828 4829 res = read_bytes(recoverfd, header2, RECOVER_ID_SIZE); 4830 if(res == -1) 4831 BAD_ERROR("Failed to read recovery file, because %s\n", 4832 strerror(errno)); 4833 if(res < RECOVER_ID_SIZE) 4834 BAD_ERROR("Recovery file appears to be truncated\n"); 4835 if(strncmp(header, header2, RECOVER_ID_SIZE) !=0 ) 4836 BAD_ERROR("Not a recovery file\n"); 4837 4838 res = read_bytes(recoverfd, &sBlk, sizeof(struct squashfs_super_block)); 4839 if(res == -1) 4840 BAD_ERROR("Failed to read recovery file, because %s\n", 4841 strerror(errno)); 4842 if(res < sizeof(struct squashfs_super_block)) 4843 BAD_ERROR("Recovery file appears to be truncated\n"); 4844 4845 res = read_fs_bytes(fd, 0, sizeof(struct squashfs_super_block), &orig_sBlk); 4846 if(res == 0) { 4847 ERROR("Failed to read superblock from output filesystem\n"); 4848 BAD_ERROR("Output filesystem is empty!\n"); 4849 } 4850 4851 if(memcmp(((char *) &sBlk) + 4, ((char *) &orig_sBlk) + 4, 4852 sizeof(struct squashfs_super_block) - 4) != 0) 4853 BAD_ERROR("Recovery file and destination file do not seem to " 4854 "match\n"); 4855 4856 bytes = sBlk.bytes_used - sBlk.inode_table_start; 4857 4858 metadata = malloc(bytes); 4859 if(metadata == NULL) 4860 MEM_ERROR(); 4861 4862 res = read_bytes(recoverfd, metadata, bytes); 4863 if(res == -1) 4864 BAD_ERROR("Failed to read recovery file, because %s\n", 4865 strerror(errno)); 4866 if(res < bytes) 4867 BAD_ERROR("Recovery file appears to be truncated\n"); 4868 4869 write_destination(fd, 0, sizeof(struct squashfs_super_block), &sBlk); 4870 4871 write_destination(fd, sBlk.inode_table_start, bytes, metadata); 4872 4873 close(recoverfd); 4874 close(fd); 4875 4876 printf("Successfully wrote recovery file \"%s\". Exiting\n", 4877 recovery_file); 4878 4879 exit(0); 4880 } 4881 4882 4883 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad) 4884 { 4885 int i; 4886 4887 sBlk->fragments = fragments; 4888 sBlk->no_ids = id_count; 4889 sBlk->inode_table_start = write_inodes(); 4890 sBlk->directory_table_start = write_directories(); 4891 sBlk->fragment_table_start = write_fragment_table(); 4892 sBlk->lookup_table_start = exportable ? write_inode_lookup_table() : 4893 SQUASHFS_INVALID_BLK; 4894 sBlk->id_table_start = write_id_table(); 4895 sBlk->xattr_id_table_start = write_xattrs(); 4896 4897 TRACE("sBlk->inode_table_start 0x%llx\n", sBlk->inode_table_start); 4898 TRACE("sBlk->directory_table_start 0x%llx\n", 4899 sBlk->directory_table_start); 4900 TRACE("sBlk->fragment_table_start 0x%llx\n", sBlk->fragment_table_start); 4901 if(exportable) 4902 TRACE("sBlk->lookup_table_start 0x%llx\n", 4903 sBlk->lookup_table_start); 4904 4905 sBlk->bytes_used = bytes; 4906 4907 sBlk->compression = comp->id; 4908 4909 SQUASHFS_INSWAP_SUPER_BLOCK(sBlk); 4910 write_destination(fd, SQUASHFS_START, sizeof(*sBlk), sBlk); 4911 4912 if(!nopad && (i = bytes & (4096 - 1))) { 4913 char temp[4096] = {0}; 4914 write_destination(fd, bytes, 4096 - i, temp); 4915 } 4916 4917 close(fd); 4918 4919 if(recovery_file) 4920 unlink(recovery_file); 4921 4922 total_bytes += total_inode_bytes + total_directory_bytes + 4923 sizeof(struct squashfs_super_block) + total_xattr_bytes; 4924 4925 printf("\n%sSquashfs %d.%d filesystem, %s compressed, data block size" 4926 " %d\n", exportable ? "Exportable " : "", SQUASHFS_MAJOR, 4927 SQUASHFS_MINOR, comp->name, block_size); 4928 printf("\t%s data, %s metadata, %s fragments, %s xattrs\n", 4929 noD ? "uncompressed" : "compressed", noI ? "uncompressed" : 4930 "compressed", no_fragments ? "no" : noF ? "uncompressed" : 4931 "compressed", no_xattrs ? "no" : noX ? "uncompressed" : 4932 "compressed"); 4933 printf("\tduplicates are %sremoved\n", duplicate_checking ? "" : 4934 "not "); 4935 printf("Filesystem size %.2f Kbytes (%.2f Mbytes)\n", bytes / 1024.0, 4936 bytes / (1024.0 * 1024.0)); 4937 printf("\t%.2f%% of uncompressed filesystem size (%.2f Kbytes)\n", 4938 ((float) bytes / total_bytes) * 100.0, total_bytes / 1024.0); 4939 printf("Inode table size %d bytes (%.2f Kbytes)\n", 4940 inode_bytes, inode_bytes / 1024.0); 4941 printf("\t%.2f%% of uncompressed inode table size (%d bytes)\n", 4942 ((float) inode_bytes / total_inode_bytes) * 100.0, 4943 total_inode_bytes); 4944 printf("Directory table size %d bytes (%.2f Kbytes)\n", 4945 directory_bytes, directory_bytes / 1024.0); 4946 printf("\t%.2f%% of uncompressed directory table size (%d bytes)\n", 4947 ((float) directory_bytes / total_directory_bytes) * 100.0, 4948 total_directory_bytes); 4949 if(total_xattr_bytes) { 4950 printf("Xattr table size %d bytes (%.2f Kbytes)\n", 4951 xattr_bytes, xattr_bytes / 1024.0); 4952 printf("\t%.2f%% of uncompressed xattr table size (%d bytes)\n", 4953 ((float) xattr_bytes / total_xattr_bytes) * 100.0, 4954 total_xattr_bytes); 4955 } 4956 if(duplicate_checking) 4957 printf("Number of duplicate files found %d\n", file_count - 4958 dup_files); 4959 else 4960 printf("No duplicate files removed\n"); 4961 printf("Number of inodes %d\n", inode_count); 4962 printf("Number of files %d\n", file_count); 4963 if(!no_fragments) 4964 printf("Number of fragments %d\n", fragments); 4965 printf("Number of symbolic links %d\n", sym_count); 4966 printf("Number of device nodes %d\n", dev_count); 4967 printf("Number of fifo nodes %d\n", fifo_count); 4968 printf("Number of socket nodes %d\n", sock_count); 4969 printf("Number of directories %d\n", dir_count); 4970 printf("Number of ids (unique uids + gids) %d\n", id_count); 4971 printf("Number of uids %d\n", uid_count); 4972 4973 for(i = 0; i < id_count; i++) { 4974 if(id_table[i]->flags & ISA_UID) { 4975 struct passwd *user = getpwuid(id_table[i]->id); 4976 printf("\t%s (%d)\n", user == NULL ? "unknown" : 4977 user->pw_name, id_table[i]->id); 4978 } 4979 } 4980 4981 printf("Number of gids %d\n", guid_count); 4982 4983 for(i = 0; i < id_count; i++) { 4984 if(id_table[i]->flags & ISA_GID) { 4985 struct group *group = getgrgid(id_table[i]->id); 4986 printf("\t%s (%d)\n", group == NULL ? "unknown" : 4987 group->gr_name, id_table[i]->id); 4988 } 4989 } 4990 } 4991 4992 4993 int parse_numberll(char *start, long long *res, int size) 4994 { 4995 char *end; 4996 long long number; 4997 4998 errno = 0; /* To distinguish success/failure after call */ 4999 5000 number = strtoll(start, &end, 10); 5001 5002 /* 5003 * check for strtoll underflow or overflow in conversion, and other 5004 * errors. 5005 */ 5006 if((errno == ERANGE && (number == LLONG_MIN || number == LLONG_MAX)) || 5007 (errno != 0 && number == 0)) 5008 return 0; 5009 5010 /* reject negative numbers as invalid */ 5011 if(number < 0) 5012 return 0; 5013 5014 if(size) { 5015 /* 5016 * Check for multiplier and trailing junk. 5017 * But first check that a number exists before the 5018 * multiplier 5019 */ 5020 if(end == start) 5021 return 0; 5022 5023 switch(end[0]) { 5024 case 'g': 5025 case 'G': 5026 if(multiply_overflowll(number, 1073741824)) 5027 return 0; 5028 number *= 1073741824; 5029 5030 if(end[1] != '\0') 5031 /* trailing junk after multiplier, but 5032 * allow it to be "bytes" */ 5033 if(strcmp(end + 1, "bytes")) 5034 return 0; 5035 5036 break; 5037 case 'm': 5038 case 'M': 5039 if(multiply_overflowll(number, 1048576)) 5040 return 0; 5041 number *= 1048576; 5042 5043 if(end[1] != '\0') 5044 /* trailing junk after multiplier, but 5045 * allow it to be "bytes" */ 5046 if(strcmp(end + 1, "bytes")) 5047 return 0; 5048 5049 break; 5050 case 'k': 5051 case 'K': 5052 if(multiply_overflowll(number, 1024)) 5053 return 0; 5054 number *= 1024; 5055 5056 if(end[1] != '\0') 5057 /* trailing junk after multiplier, but 5058 * allow it to be "bytes" */ 5059 if(strcmp(end + 1, "bytes")) 5060 return 0; 5061 5062 break; 5063 case '\0': 5064 break; 5065 default: 5066 /* trailing junk after number */ 5067 return 0; 5068 } 5069 } else if(end[0] != '\0') 5070 /* trailing junk after number */ 5071 return 0; 5072 5073 *res = number; 5074 return 1; 5075 } 5076 5077 5078 int parse_number(char *start, int *res, int size) 5079 { 5080 long long number; 5081 5082 if(!parse_numberll(start, &number, size)) 5083 return 0; 5084 5085 /* check if long result will overflow signed int */ 5086 if(number > INT_MAX) 5087 return 0; 5088 5089 *res = (int) number; 5090 return 1; 5091 } 5092 5093 5094 int parse_num(char *arg, int *res) 5095 { 5096 return parse_number(arg, res, 0); 5097 } 5098 5099 5100 int get_physical_memory() 5101 { 5102 int phys_mem; 5103 #ifndef linux 5104 #ifdef HW_MEMSIZE 5105 #define SYSCTL_PHYSMEM HW_MEMSIZE 5106 #elif defined(HW_PHYSMEM64) 5107 #define SYSCTL_PHYSMEM HW_PHYSMEM64 5108 #else 5109 #define SYSCTL_PHYSMEM HW_PHYSMEM 5110 #endif 5111 5112 int mib[2]; 5113 uint64_t sysctl_physmem = 0; 5114 size_t sysctl_len = sizeof(sysctl_physmem); 5115 5116 mib[0] = CTL_HW; 5117 mib[1] = SYSCTL_PHYSMEM; 5118 5119 if(sysctl(mib, 2, &sysctl_physmem, &sysctl_len, NULL, 0) == 0) { 5120 /* some systems use 32-bit values, work with what we're given */ 5121 if (sysctl_len == 4) 5122 sysctl_physmem = *(uint32_t*)&sysctl_physmem; 5123 phys_mem = sysctl_physmem >> 20; 5124 } else { 5125 ERROR_START("Failed to get amount of available " 5126 "memory."); 5127 ERROR_EXIT(" Defaulting to least viable amount\n"); 5128 phys_mem = SQUASHFS_LOWMEM; 5129 } 5130 #undef SYSCTL_PHYSMEM 5131 #else 5132 /* Long longs are used here because with PAE, a 32-bit 5133 machine can have more than 4GB of physical memory */ 5134 5135 long long num_pages = sysconf(_SC_PHYS_PAGES); 5136 long long page_size = sysconf(_SC_PAGESIZE); 5137 phys_mem = num_pages * page_size >> 20; 5138 if(num_pages == -1 || page_size == -1) 5139 return 0; 5140 5141 #endif 5142 5143 if(phys_mem < SQUASHFS_LOWMEM) 5144 BAD_ERROR("Mksquashfs requires more physical memory than is " 5145 "available!\n"); 5146 5147 return phys_mem; 5148 } 5149 5150 5151 void check_usable_phys_mem(int total_mem) 5152 { 5153 /* 5154 * We want to allow users to use as much of their physical 5155 * memory as they wish. However, for practical reasons there are 5156 * limits which need to be imposed, to protect users from themselves 5157 * and to prevent people from using Mksquashfs as a DOS attack by using 5158 * all physical memory. Mksquashfs uses memory to cache data from disk 5159 * to optimise performance. It is pointless to ask it to use more 5160 * than 75% of physical memory, as this causes thrashing and it is thus 5161 * self-defeating. 5162 */ 5163 int mem = get_physical_memory(); 5164 5165 mem = (mem >> 1) + (mem >> 2); /* 75% */ 5166 5167 if(total_mem > mem && mem) { 5168 ERROR("Total memory requested is more than 75%% of physical " 5169 "memory.\n"); 5170 ERROR("Mksquashfs uses memory to cache data from disk to " 5171 "optimise performance.\n"); 5172 ERROR("It is pointless to ask it to use more than this amount " 5173 "of memory, as this\n"); 5174 ERROR("causes thrashing and it is thus self-defeating.\n"); 5175 BAD_ERROR("Requested memory size too large\n"); 5176 } 5177 5178 if(sizeof(void *) == 4 && total_mem > 2048) { 5179 /* 5180 * If we're running on a kernel with PAE or on a 64-bit kernel, 5181 * then the 75% physical memory limit can still easily exceed 5182 * the addressable memory by this process. 5183 * 5184 * Due to the typical kernel/user-space split (1GB/3GB, or 5185 * 2GB/2GB), we have to conservatively assume the 32-bit 5186 * processes can only address 2-3GB. So refuse if the user 5187 * tries to allocate more than 2GB. 5188 */ 5189 ERROR("Total memory requested may exceed maximum " 5190 "addressable memory by this process\n"); 5191 BAD_ERROR("Requested memory size too large\n"); 5192 } 5193 } 5194 5195 5196 int get_default_phys_mem() 5197 { 5198 /* 5199 * get_physical_memory() relies on /proc being mounted. 5200 * If it fails, issue a warning, and use 5201 * SQUASHFS_LOWMEM / SQUASHFS_TAKE as default, 5202 * and allow a larger value to be set with -mem. 5203 */ 5204 int mem = get_physical_memory(); 5205 5206 if(mem == 0) { 5207 mem = SQUASHFS_LOWMEM / SQUASHFS_TAKE; 5208 5209 ERROR("Warning: Cannot get size of physical memory, probably " 5210 "because /proc is missing.\n"); 5211 ERROR("Warning: Defaulting to minimal use of %d Mbytes, use " 5212 "-mem to set a better value,\n", mem); 5213 ERROR("Warning: or fix /proc.\n"); 5214 } else 5215 mem /= SQUASHFS_TAKE; 5216 5217 if(sizeof(void *) == 4 && mem > 640) { 5218 /* 5219 * If we're running on a kernel with PAE or on a 64-bit kernel, 5220 * the default memory usage can exceed the addressable 5221 * memory by this process. 5222 * Due to the typical kernel/user-space split (1GB/3GB, or 5223 * 2GB/2GB), we have to conservatively assume the 32-bit 5224 * processes can only address 2-3GB. So limit the default 5225 * usage to 640M, which gives room for other data. 5226 */ 5227 mem = 640; 5228 } 5229 5230 return mem; 5231 } 5232 5233 5234 void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq, 5235 int *fwriteq) 5236 { 5237 *readq = mem / SQUASHFS_READQ_MEM; 5238 *bwriteq = mem / SQUASHFS_BWRITEQ_MEM; 5239 *fwriteq = mem / SQUASHFS_FWRITEQ_MEM; 5240 *fragq = mem - *readq - *bwriteq - *fwriteq; 5241 } 5242 5243 5244 #define VERSION() \ 5245 printf("mksquashfs version 4.3-git (2014/09/12)\n");\ 5246 printf("copyright (C) 2014 Phillip Lougher "\ 5247 "<phillip (at) squashfs.org.uk>\n\n"); \ 5248 printf("This program is free software; you can redistribute it and/or"\ 5249 "\n");\ 5250 printf("modify it under the terms of the GNU General Public License"\ 5251 "\n");\ 5252 printf("as published by the Free Software Foundation; either version "\ 5253 "2,\n");\ 5254 printf("or (at your option) any later version.\n\n");\ 5255 printf("This program is distributed in the hope that it will be "\ 5256 "useful,\n");\ 5257 printf("but WITHOUT ANY WARRANTY; without even the implied warranty "\ 5258 "of\n");\ 5259 printf("MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the"\ 5260 "\n");\ 5261 printf("GNU General Public License for more details.\n"); 5262 int main(int argc, char *argv[]) 5263 { 5264 struct stat buf, source_buf; 5265 int res, i; 5266 char *b, *root_name = NULL; 5267 int keep_as_directory = FALSE; 5268 squashfs_inode inode; 5269 int readq; 5270 int fragq; 5271 int bwriteq; 5272 int fwriteq; 5273 int total_mem = get_default_phys_mem(); 5274 int progress = TRUE; 5275 int force_progress = FALSE; 5276 struct file_buffer **fragment = NULL; 5277 /* ANDROID CHANGES START*/ 5278 #ifdef ANDROID 5279 const char *fs_config_file = NULL; 5280 #endif 5281 /* ANDROID CHANGES END */ 5282 5283 if(argc > 1 && strcmp(argv[1], "-version") == 0) { 5284 VERSION(); 5285 exit(0); 5286 } 5287 5288 block_log = slog(block_size); 5289 calculate_queue_sizes(total_mem, &readq, &fragq, &bwriteq, &fwriteq); 5290 5291 for(i = 1; i < argc && argv[i][0] != '-'; i++); 5292 if(i < 3) 5293 goto printOptions; 5294 source_path = argv + 1; 5295 source = i - 2; 5296 5297 /* 5298 * Scan the command line for -comp xxx option, this is to ensure 5299 * any -X compressor specific options are passed to the 5300 * correct compressor 5301 */ 5302 for(; i < argc; i++) { 5303 struct compressor *prev_comp = comp; 5304 5305 if(strcmp(argv[i], "-comp") == 0) { 5306 if(++i == argc) { 5307 ERROR("%s: -comp missing compression type\n", 5308 argv[0]); 5309 exit(1); 5310 } 5311 comp = lookup_compressor(argv[i]); 5312 if(!comp->supported) { 5313 ERROR("%s: Compressor \"%s\" is not supported!" 5314 "\n", argv[0], argv[i]); 5315 ERROR("%s: Compressors available:\n", argv[0]); 5316 display_compressors("", COMP_DEFAULT); 5317 exit(1); 5318 } 5319 if(prev_comp != NULL && prev_comp != comp) { 5320 ERROR("%s: -comp multiple conflicting -comp" 5321 " options specified on command line" 5322 ", previously %s, now %s\n", argv[0], 5323 prev_comp->name, comp->name); 5324 exit(1); 5325 } 5326 compressor_opt_parsed = 1; 5327 5328 } else if(strcmp(argv[i], "-e") == 0) 5329 break; 5330 else if(strcmp(argv[i], "-root-becomes") == 0 || 5331 strcmp(argv[i], "-ef") == 0 || 5332 strcmp(argv[i], "-pf") == 0 || 5333 strcmp(argv[i], "-vaf") == 0 || 5334 strcmp(argv[i], "-comp") == 0) 5335 i++; 5336 } 5337 5338 /* 5339 * if no -comp option specified lookup default compressor. Note the 5340 * Makefile ensures the default compressor has been built, and so we 5341 * don't need to to check for failure here 5342 */ 5343 if(comp == NULL) 5344 comp = lookup_compressor(COMP_DEFAULT); 5345 5346 for(i = source + 2; i < argc; i++) { 5347 if(strcmp(argv[i], "-action") == 0 || 5348 strcmp(argv[i], "-a") ==0) { 5349 if(++i == argc) { 5350 ERROR("%s: %s missing action\n", 5351 argv[0], argv[i - 1]); 5352 exit(1); 5353 } 5354 res = parse_action(argv[i], ACTION_LOG_NONE); 5355 if(res == 0) 5356 exit(1); 5357 5358 } else if(strcmp(argv[i], "-verbose-action") == 0 || 5359 strcmp(argv[i], "-va") ==0) { 5360 if(++i == argc) { 5361 ERROR("%s: %s missing action\n", 5362 argv[0], argv[i - 1]); 5363 exit(1); 5364 } 5365 res = parse_action(argv[i], ACTION_LOG_VERBOSE); 5366 if(res == 0) 5367 exit(1); 5368 5369 } else if(strcmp(argv[i], "-true-action") == 0 || 5370 strcmp(argv[i], "-ta") ==0) { 5371 if(++i == argc) { 5372 ERROR("%s: %s missing action\n", 5373 argv[0], argv[i - 1]); 5374 exit(1); 5375 } 5376 res = parse_action(argv[i], ACTION_LOG_TRUE); 5377 if(res == 0) 5378 exit(1); 5379 5380 } else if(strcmp(argv[i], "-false-action") == 0 || 5381 strcmp(argv[i], "-fa") ==0) { 5382 if(++i == argc) { 5383 ERROR("%s: %s missing action\n", 5384 argv[0], argv[i - 1]); 5385 exit(1); 5386 } 5387 res = parse_action(argv[i], ACTION_LOG_FALSE); 5388 if(res == 0) 5389 exit(1); 5390 5391 } else if(strcmp(argv[i], "-action-file") == 0 || 5392 strcmp(argv[i], "-af") ==0) { 5393 if(++i == argc) { 5394 ERROR("%s: %s missing filename\n", argv[0], 5395 argv[i - 1]); 5396 exit(1); 5397 } 5398 if(read_action_file(argv[i], ACTION_LOG_NONE) == FALSE) 5399 exit(1); 5400 5401 } else if(strcmp(argv[i], "-verbose-action-file") == 0 || 5402 strcmp(argv[i], "-vaf") ==0) { 5403 if(++i == argc) { 5404 ERROR("%s: %s missing filename\n", argv[0], 5405 argv[i - 1]); 5406 exit(1); 5407 } 5408 if(read_action_file(argv[i], ACTION_LOG_VERBOSE) == FALSE) 5409 exit(1); 5410 5411 } else if(strcmp(argv[i], "-true-action-file") == 0 || 5412 strcmp(argv[i], "-taf") ==0) { 5413 if(++i == argc) { 5414 ERROR("%s: %s missing filename\n", argv[0], 5415 argv[i - 1]); 5416 exit(1); 5417 } 5418 if(read_action_file(argv[i], ACTION_LOG_TRUE) == FALSE) 5419 exit(1); 5420 5421 } else if(strcmp(argv[i], "-false-action-file") == 0 || 5422 strcmp(argv[i], "-faf") ==0) { 5423 if(++i == argc) { 5424 ERROR("%s: %s missing filename\n", argv[0], 5425 argv[i - 1]); 5426 exit(1); 5427 } 5428 if(read_action_file(argv[i], ACTION_LOG_FALSE) == FALSE) 5429 exit(1); 5430 5431 } else if(strcmp(argv[i], "-comp") == 0) 5432 /* parsed previously */ 5433 i++; 5434 5435 else if(strncmp(argv[i], "-X", 2) == 0) { 5436 int args; 5437 5438 if(strcmp(argv[i] + 2, "help") == 0) 5439 goto print_compressor_options; 5440 5441 args = compressor_options(comp, argv + i, argc - i); 5442 if(args < 0) { 5443 if(args == -1) { 5444 ERROR("%s: Unrecognised compressor" 5445 " option %s\n", argv[0], 5446 argv[i]); 5447 if(!compressor_opt_parsed) 5448 ERROR("%s: Did you forget to" 5449 " specify -comp?\n", 5450 argv[0]); 5451 print_compressor_options: 5452 ERROR("%s: selected compressor \"%s\"" 5453 ". Options supported: %s\n", 5454 argv[0], comp->name, 5455 comp->usage ? "" : "none"); 5456 if(comp->usage) 5457 comp->usage(); 5458 } 5459 exit(1); 5460 } 5461 i += args; 5462 5463 } else if(strcmp(argv[i], "-pf") == 0) { 5464 if(++i == argc) { 5465 ERROR("%s: -pf missing filename\n", argv[0]); 5466 exit(1); 5467 } 5468 if(read_pseudo_file(argv[i]) == FALSE) 5469 exit(1); 5470 } else if(strcmp(argv[i], "-p") == 0) { 5471 if(++i == argc) { 5472 ERROR("%s: -p missing pseudo file definition\n", 5473 argv[0]); 5474 exit(1); 5475 } 5476 if(read_pseudo_def(argv[i]) == FALSE) 5477 exit(1); 5478 } else if(strcmp(argv[i], "-recover") == 0) { 5479 if(++i == argc) { 5480 ERROR("%s: -recover missing recovery file\n", 5481 argv[0]); 5482 exit(1); 5483 } 5484 read_recovery_data(argv[i], argv[source + 1]); 5485 } else if(strcmp(argv[i], "-no-recovery") == 0) 5486 recover = FALSE; 5487 else if(strcmp(argv[i], "-wildcards") == 0) { 5488 old_exclude = FALSE; 5489 use_regex = FALSE; 5490 } else if(strcmp(argv[i], "-regex") == 0) { 5491 old_exclude = FALSE; 5492 use_regex = TRUE; 5493 } else if(strcmp(argv[i], "-no-sparse") == 0) 5494 sparse_files = FALSE; 5495 else if(strcmp(argv[i], "-no-progress") == 0) 5496 progress = FALSE; 5497 else if(strcmp(argv[i], "-progress") == 0) 5498 force_progress = TRUE; 5499 else if(strcmp(argv[i], "-no-exports") == 0) 5500 exportable = FALSE; 5501 else if(strcmp(argv[i], "-processors") == 0) { 5502 if((++i == argc) || !parse_num(argv[i], &processors)) { 5503 ERROR("%s: -processors missing or invalid " 5504 "processor number\n", argv[0]); 5505 exit(1); 5506 } 5507 if(processors < 1) { 5508 ERROR("%s: -processors should be 1 or larger\n", 5509 argv[0]); 5510 exit(1); 5511 } 5512 } else if(strcmp(argv[i], "-read-queue") == 0) { 5513 if((++i == argc) || !parse_num(argv[i], &readq)) { 5514 ERROR("%s: -read-queue missing or invalid " 5515 "queue size\n", argv[0]); 5516 exit(1); 5517 } 5518 if(readq < 1) { 5519 ERROR("%s: -read-queue should be 1 megabyte or " 5520 "larger\n", argv[0]); 5521 exit(1); 5522 } 5523 } else if(strcmp(argv[i], "-write-queue") == 0) { 5524 if((++i == argc) || !parse_num(argv[i], &bwriteq)) { 5525 ERROR("%s: -write-queue missing or invalid " 5526 "queue size\n", argv[0]); 5527 exit(1); 5528 } 5529 if(bwriteq < 2) { 5530 ERROR("%s: -write-queue should be 2 megabytes " 5531 "or larger\n", argv[0]); 5532 exit(1); 5533 } 5534 fwriteq = bwriteq >> 1; 5535 bwriteq -= fwriteq; 5536 } else if(strcmp(argv[i], "-fragment-queue") == 0) { 5537 if((++i == argc) || !parse_num(argv[i], &fragq)) { 5538 ERROR("%s: -fragment-queue missing or invalid " 5539 "queue size\n", argv[0]); 5540 exit(1); 5541 } 5542 if(fragq < 1) { 5543 ERROR("%s: -fragment-queue should be 1 " 5544 "megabyte or larger\n", argv[0]); 5545 exit(1); 5546 } 5547 } else if(strcmp(argv[i], "-mem") == 0) { 5548 long long number; 5549 5550 if((++i == argc) || 5551 !parse_numberll(argv[i], &number, 1)) { 5552 ERROR("%s: -mem missing or invalid mem size\n", 5553 argv[0]); 5554 exit(1); 5555 } 5556 5557 /* 5558 * convert from bytes to Mbytes, ensuring the value 5559 * does not overflow a signed int 5560 */ 5561 if(number >= (1LL << 51)) { 5562 ERROR("%s: -mem invalid mem size\n", argv[0]); 5563 exit(1); 5564 } 5565 5566 total_mem = number / 1048576; 5567 if(total_mem < (SQUASHFS_LOWMEM / SQUASHFS_TAKE)) { 5568 ERROR("%s: -mem should be %d Mbytes or " 5569 "larger\n", argv[0], 5570 SQUASHFS_LOWMEM / SQUASHFS_TAKE); 5571 exit(1); 5572 } 5573 calculate_queue_sizes(total_mem, &readq, &fragq, 5574 &bwriteq, &fwriteq); 5575 } else if(strcmp(argv[i], "-b") == 0) { 5576 if(++i == argc) { 5577 ERROR("%s: -b missing block size\n", argv[0]); 5578 exit(1); 5579 } 5580 if(!parse_number(argv[i], &block_size, 1)) { 5581 ERROR("%s: -b invalid block size\n", argv[0]); 5582 exit(1); 5583 } 5584 if((block_log = slog(block_size)) == 0) { 5585 ERROR("%s: -b block size not power of two or " 5586 "not between 4096 and 1Mbyte\n", 5587 argv[0]); 5588 exit(1); 5589 } 5590 } else if(strcmp(argv[i], "-ef") == 0) { 5591 if(++i == argc) { 5592 ERROR("%s: -ef missing filename\n", argv[0]); 5593 exit(1); 5594 } 5595 } else if(strcmp(argv[i], "-no-duplicates") == 0) 5596 duplicate_checking = FALSE; 5597 5598 else if(strcmp(argv[i], "-no-fragments") == 0) 5599 no_fragments = TRUE; 5600 5601 else if(strcmp(argv[i], "-always-use-fragments") == 0) 5602 always_use_fragments = TRUE; 5603 5604 else if(strcmp(argv[i], "-sort") == 0) { 5605 if(++i == argc) { 5606 ERROR("%s: -sort missing filename\n", argv[0]); 5607 exit(1); 5608 } 5609 } else if(strcmp(argv[i], "-all-root") == 0 || 5610 strcmp(argv[i], "-root-owned") == 0) 5611 global_uid = global_gid = 0; 5612 5613 else if(strcmp(argv[i], "-force-uid") == 0) { 5614 if(++i == argc) { 5615 ERROR("%s: -force-uid missing uid or user\n", 5616 argv[0]); 5617 exit(1); 5618 } 5619 if((global_uid = strtoll(argv[i], &b, 10)), *b =='\0') { 5620 if(global_uid < 0 || global_uid > 5621 (((long long) 1 << 32) - 1)) { 5622 ERROR("%s: -force-uid uid out of range" 5623 "\n", argv[0]); 5624 exit(1); 5625 } 5626 } else { 5627 struct passwd *uid = getpwnam(argv[i]); 5628 if(uid) 5629 global_uid = uid->pw_uid; 5630 else { 5631 ERROR("%s: -force-uid invalid uid or " 5632 "unknown user\n", argv[0]); 5633 exit(1); 5634 } 5635 } 5636 } else if(strcmp(argv[i], "-force-gid") == 0) { 5637 if(++i == argc) { 5638 ERROR("%s: -force-gid missing gid or group\n", 5639 argv[0]); 5640 exit(1); 5641 } 5642 if((global_gid = strtoll(argv[i], &b, 10)), *b =='\0') { 5643 if(global_gid < 0 || global_gid > 5644 (((long long) 1 << 32) - 1)) { 5645 ERROR("%s: -force-gid gid out of range" 5646 "\n", argv[0]); 5647 exit(1); 5648 } 5649 } else { 5650 struct group *gid = getgrnam(argv[i]); 5651 if(gid) 5652 global_gid = gid->gr_gid; 5653 else { 5654 ERROR("%s: -force-gid invalid gid or " 5655 "unknown group\n", argv[0]); 5656 exit(1); 5657 } 5658 } 5659 } else if(strcmp(argv[i], "-noI") == 0 || 5660 strcmp(argv[i], "-noInodeCompression") == 0) 5661 noI = TRUE; 5662 5663 else if(strcmp(argv[i], "-noD") == 0 || 5664 strcmp(argv[i], "-noDataCompression") == 0) 5665 noD = TRUE; 5666 5667 else if(strcmp(argv[i], "-noF") == 0 || 5668 strcmp(argv[i], "-noFragmentCompression") == 0) 5669 noF = TRUE; 5670 5671 else if(strcmp(argv[i], "-noX") == 0 || 5672 strcmp(argv[i], "-noXattrCompression") == 0) 5673 noX = TRUE; 5674 5675 else if(strcmp(argv[i], "-no-xattrs") == 0) 5676 no_xattrs = TRUE; 5677 5678 else if(strcmp(argv[i], "-xattrs") == 0) 5679 no_xattrs = FALSE; 5680 5681 /* ANDROID CHANGES START*/ 5682 #ifdef ANDROID 5683 else if(strcmp(argv[i], "-context-file") == 0) { 5684 if(++i == argc) { 5685 ERROR("%s: -context-file: missing file name\n", 5686 argv[0]); 5687 exit(1); 5688 } 5689 context_file = argv[i]; 5690 } 5691 else if(strcmp(argv[i], "-fs-config-file") == 0) { 5692 if(++i == argc) { 5693 ERROR("%s: -fs-config-file: missing file name\n", 5694 argv[0]); 5695 exit(1); 5696 } 5697 fs_config_file = argv[i]; 5698 } 5699 #endif 5700 /* ANDROID CHANGES END */ 5701 else if(strcmp(argv[i], "-nopad") == 0) 5702 nopad = TRUE; 5703 5704 else if(strcmp(argv[i], "-info") == 0) 5705 silent = FALSE; 5706 5707 else if(strcmp(argv[i], "-e") == 0) 5708 break; 5709 5710 else if(strcmp(argv[i], "-noappend") == 0) 5711 delete = TRUE; 5712 5713 else if(strcmp(argv[i], "-keep-as-directory") == 0) 5714 keep_as_directory = TRUE; 5715 /* ANDROID CHANGES START*/ 5716 #ifdef ANDROID 5717 else if(strcmp(argv[i], "-android-fs-config") == 0) 5718 android_config = TRUE; 5719 else if(strcmp(argv[i], "-mount-point") == 0) { 5720 if(++i == argc) { 5721 ERROR("%s: -mount-point: missing mount point name\n", 5722 argv[0]); 5723 exit(1); 5724 } 5725 mount_point = argv[i]; 5726 } 5727 else if(strcmp(argv[i], "-product-out") == 0) { 5728 if(++i == argc) { 5729 ERROR("%s: -product-out: missing path name\n", 5730 argv[0]); 5731 exit(1); 5732 } 5733 target_out_path = argv[i]; 5734 } 5735 else if(strcmp(argv[i], "-disable-4k-align") == 0) 5736 align_4k_blocks = FALSE; 5737 else if(strcmp(argv[i], "-block-map") == 0) { 5738 if(++i == argc) { 5739 ERROR("%s: -block-map: missing path name\n", 5740 argv[0]); 5741 exit(1); 5742 } 5743 block_map_file = fopen(argv[i], "w"); 5744 if (block_map_file == NULL) { 5745 ERROR("%s: -block-map: failed to open %s\n", 5746 argv[0], argv[i]); 5747 exit(1); 5748 } 5749 if (!align_4k_blocks) { 5750 ERROR("WARNING: Using block maps with unaligned 4k blocks " 5751 "is not ideal as block map offsets are multiples of 4k, " 5752 "consider not passing -disable-4k-align\n"); 5753 } 5754 } 5755 #endif 5756 /* ANDROID CHANGES END */ 5757 5758 else if(strcmp(argv[i], "-exit-on-error") == 0) 5759 exit_on_error = TRUE; 5760 5761 else if(strcmp(argv[i], "-root-becomes") == 0) { 5762 if(++i == argc) { 5763 ERROR("%s: -root-becomes: missing name\n", 5764 argv[0]); 5765 exit(1); 5766 } 5767 root_name = argv[i]; 5768 } else if(strcmp(argv[i], "-version") == 0) { 5769 VERSION(); 5770 } else { 5771 ERROR("%s: invalid option\n\n", argv[0]); 5772 printOptions: 5773 ERROR("SYNTAX:%s source1 source2 ... dest [options] " 5774 "[-e list of exclude\ndirs/files]\n", argv[0]); 5775 ERROR("\nFilesystem build options:\n"); 5776 ERROR("-comp <comp>\t\tselect <comp> compression\n"); 5777 ERROR("\t\t\tCompressors available:\n"); 5778 display_compressors("\t\t\t", COMP_DEFAULT); 5779 ERROR("-b <block_size>\t\tset data block to " 5780 "<block_size>. Default 128 Kbytes\n"); 5781 ERROR("\t\t\tOptionally a suffix of K or M can be" 5782 " given to specify\n\t\t\tKbytes or Mbytes" 5783 " respectively\n"); 5784 ERROR("-no-exports\t\tdon't make the filesystem " 5785 "exportable via NFS\n"); 5786 ERROR("-no-sparse\t\tdon't detect sparse files\n"); 5787 ERROR("-no-xattrs\t\tdon't store extended attributes" 5788 NOXOPT_STR "\n"); 5789 ERROR("-xattrs\t\t\tstore extended attributes" XOPT_STR 5790 "\n"); 5791 /* ANDROID CHANGES START*/ 5792 #ifdef ANDROID 5793 ERROR("-context-file <file>\tApply selinux security " 5794 "xattrs from context-file instead\n\t\t\t" 5795 "of reading xattrs from file system\n"); 5796 ERROR("-fs-config-file <file>\tAndroid specific " 5797 "filesystem config file\n"); 5798 #endif 5799 /* ANDROID CHANGES END */ 5800 ERROR("-noI\t\t\tdo not compress inode table\n"); 5801 ERROR("-noD\t\t\tdo not compress data blocks\n"); 5802 ERROR("-noF\t\t\tdo not compress fragment blocks\n"); 5803 ERROR("-noX\t\t\tdo not compress extended " 5804 "attributes\n"); 5805 ERROR("-no-fragments\t\tdo not use fragments\n"); 5806 ERROR("-always-use-fragments\tuse fragment blocks for " 5807 "files larger than block size\n"); 5808 ERROR("-no-duplicates\t\tdo not perform duplicate " 5809 "checking\n"); 5810 ERROR("-all-root\t\tmake all files owned by root\n"); 5811 ERROR("-force-uid uid\t\tset all file uids to uid\n"); 5812 ERROR("-force-gid gid\t\tset all file gids to gid\n"); 5813 ERROR("-nopad\t\t\tdo not pad filesystem to a multiple " 5814 "of 4K\n"); 5815 ERROR("-keep-as-directory\tif one source directory is " 5816 "specified, create a root\n"); 5817 ERROR("\t\t\tdirectory containing that directory, " 5818 "rather than the\n"); 5819 ERROR("\t\t\tcontents of the directory\n"); 5820 /* ANDROID CHANGES START*/ 5821 #ifdef ANDROID 5822 ERROR("-android-fs-config\tuse android fs config " 5823 "for mode, uid, and gids of inodes\n"); 5824 ERROR("-mount-point <name>\tNeed to be provided when " 5825 "android-fs-config or context-file\n\t\t\tare " 5826 "enabled and source directory is not mount point\n"); 5827 ERROR("-product-out <path>\tPRODUCT_OUT directory to " 5828 "read device specific FS rules files from\n"); 5829 ERROR("-disable-4k-align \tDon't 4k align data blocks. Default is false\n"); 5830 ERROR("-block-map <path>\tGenerate a block map for non-fragment files\n"); 5831 #endif 5832 /* ANDROID CHANGES END */ 5833 ERROR("\nFilesystem filter options:\n"); 5834 ERROR("-p <pseudo-definition>\tAdd pseudo file " 5835 "definition\n"); 5836 ERROR("-pf <pseudo-file>\tAdd list of pseudo file " 5837 "definitions\n"); 5838 ERROR("-sort <sort_file>\tsort files according to " 5839 "priorities in <sort_file>. One\n"); 5840 ERROR("\t\t\tfile or dir with priority per line. " 5841 "Priority -32768 to\n"); 5842 ERROR("\t\t\t32767, default priority 0\n"); 5843 ERROR("-ef <exclude_file>\tlist of exclude dirs/files." 5844 " One per line\n"); 5845 ERROR("-wildcards\t\tAllow extended shell wildcards " 5846 "(globbing) to be used in\n\t\t\texclude " 5847 "dirs/files\n"); 5848 ERROR("-regex\t\t\tAllow POSIX regular expressions to " 5849 "be used in exclude\n\t\t\tdirs/files\n"); 5850 ERROR("\nFilesystem append options:\n"); 5851 ERROR("-noappend\t\tdo not append to existing " 5852 "filesystem\n"); 5853 ERROR("-root-becomes <name>\twhen appending source " 5854 "files/directories, make the\n"); 5855 ERROR("\t\t\toriginal root become a subdirectory in " 5856 "the new root\n"); 5857 ERROR("\t\t\tcalled <name>, rather than adding the new " 5858 "source items\n"); 5859 ERROR("\t\t\tto the original root\n"); 5860 ERROR("\nMksquashfs runtime options:\n"); 5861 ERROR("-version\t\tprint version, licence and " 5862 "copyright message\n"); 5863 ERROR("-exit-on-error\t\ttreat normally ignored errors " 5864 "as fatal\n"); 5865 ERROR("-recover <name>\t\trecover filesystem data " 5866 "using recovery file <name>\n"); 5867 ERROR("-no-recovery\t\tdon't generate a recovery " 5868 "file\n"); 5869 ERROR("-info\t\t\tprint files written to filesystem\n"); 5870 ERROR("-no-progress\t\tdon't display the progress " 5871 "bar\n"); 5872 ERROR("-progress\t\tdisplay progress bar when using " 5873 "the -info option\n"); 5874 ERROR("-processors <number>\tUse <number> processors." 5875 " By default will use number of\n"); 5876 ERROR("\t\t\tprocessors available\n"); 5877 ERROR("-mem <size>\t\tUse <size> physical memory. " 5878 "Currently set to %dM\n", total_mem); 5879 ERROR("\t\t\tOptionally a suffix of K, M or G can be" 5880 " given to specify\n\t\t\tKbytes, Mbytes or" 5881 " Gbytes respectively\n"); 5882 ERROR("\nMiscellaneous options:\n"); 5883 ERROR("-root-owned\t\talternative name for -all-root" 5884 "\n"); 5885 ERROR("-noInodeCompression\talternative name for -noI" 5886 "\n"); 5887 ERROR("-noDataCompression\talternative name for -noD" 5888 "\n"); 5889 ERROR("-noFragmentCompression\talternative name for " 5890 "-noF\n"); 5891 ERROR("-noXattrCompression\talternative name for " 5892 "-noX\n"); 5893 ERROR("\n-Xhelp\t\t\tprint compressor options for" 5894 " selected compressor\n"); 5895 ERROR("\nCompressors available and compressor specific " 5896 "options:\n"); 5897 display_compressor_usage(COMP_DEFAULT); 5898 exit(1); 5899 } 5900 } 5901 5902 /* ANDROID CHANGES START*/ 5903 #ifdef ANDROID 5904 if (fs_config_file) { 5905 if (load_canned_fs_config(fs_config_file) < 0) { 5906 fprintf(stderr, "failed to load %s\n", fs_config_file); 5907 exit(1); 5908 } 5909 fs_config_func = canned_fs_config; 5910 } else if (mount_point) { 5911 fs_config_func = fs_config; 5912 } 5913 #endif 5914 /* ANDROID CHANGES END */ 5915 5916 /* 5917 * Some compressors may need the options to be checked for validity 5918 * once all the options have been processed 5919 */ 5920 res = compressor_options_post(comp, block_size); 5921 if(res) 5922 EXIT_MKSQUASHFS(); 5923 5924 /* 5925 * If the -info option has been selected then disable the 5926 * progress bar unless it has been explicitly enabled with 5927 * the -progress option 5928 */ 5929 if(!silent) 5930 progress = force_progress; 5931 5932 #ifdef SQUASHFS_TRACE 5933 /* 5934 * Disable progress bar if full debug tracing is enabled. 5935 * The progress bar in this case just gets in the way of the 5936 * debug trace output 5937 */ 5938 progress = FALSE; 5939 #endif 5940 5941 for(i = 0; i < source; i++) 5942 if(lstat(source_path[i], &source_buf) == -1) { 5943 fprintf(stderr, "Cannot stat source directory \"%s\" " 5944 "because %s\n", source_path[i], 5945 strerror(errno)); 5946 EXIT_MKSQUASHFS(); 5947 } 5948 5949 destination_file = argv[source + 1]; 5950 if(stat(argv[source + 1], &buf) == -1) { 5951 if(errno == ENOENT) { /* Does not exist */ 5952 fd = open(argv[source + 1], O_CREAT | O_TRUNC | O_RDWR, 5953 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); 5954 if(fd == -1) { 5955 perror("Could not create destination file"); 5956 exit(1); 5957 } 5958 delete = TRUE; 5959 } else { 5960 perror("Could not stat destination file"); 5961 exit(1); 5962 } 5963 5964 } else { 5965 if(S_ISBLK(buf.st_mode)) { 5966 if((fd = open(argv[source + 1], O_RDWR)) == -1) { 5967 perror("Could not open block device as " 5968 "destination"); 5969 exit(1); 5970 } 5971 block_device = 1; 5972 5973 } else if(S_ISREG(buf.st_mode)) { 5974 fd = open(argv[source + 1], (delete ? O_TRUNC : 0) | 5975 O_RDWR); 5976 if(fd == -1) { 5977 perror("Could not open regular file for " 5978 "writing as destination"); 5979 exit(1); 5980 } 5981 } 5982 else { 5983 ERROR("Destination not block device or regular file\n"); 5984 exit(1); 5985 } 5986 5987 } 5988 5989 /* 5990 * process the exclude files - must be done afer destination file has 5991 * been possibly created 5992 */ 5993 for(i = source + 2; i < argc; i++) 5994 if(strcmp(argv[i], "-ef") == 0) 5995 /* 5996 * Note presence of filename arg has already 5997 * been checked 5998 */ 5999 process_exclude_file(argv[++i]); 6000 else if(strcmp(argv[i], "-e") == 0) 6001 break; 6002 else if(strcmp(argv[i], "-root-becomes") == 0 || 6003 strcmp(argv[i], "-sort") == 0 || 6004 strcmp(argv[i], "-pf") == 0 || 6005 strcmp(argv[i], "-af") == 0 || 6006 strcmp(argv[i], "-vaf") == 0 || 6007 strcmp(argv[i], "-comp") == 0) 6008 i++; 6009 6010 if(i != argc) { 6011 if(++i == argc) { 6012 ERROR("%s: -e missing arguments\n", argv[0]); 6013 EXIT_MKSQUASHFS(); 6014 } 6015 while(i < argc) 6016 if(old_exclude) 6017 old_add_exclude(argv[i++]); 6018 else 6019 add_exclude(argv[i++]); 6020 } 6021 6022 /* process the sort files - must be done afer the exclude files */ 6023 for(i = source + 2; i < argc; i++) 6024 if(strcmp(argv[i], "-sort") == 0) { 6025 int res = read_sort_file(argv[++i], source, 6026 source_path); 6027 if(res == FALSE) 6028 BAD_ERROR("Failed to read sort file\n"); 6029 sorted ++; 6030 } else if(strcmp(argv[i], "-e") == 0) 6031 break; 6032 else if(strcmp(argv[i], "-root-becomes") == 0 || 6033 strcmp(argv[i], "-ef") == 0 || 6034 strcmp(argv[i], "-pf") == 0 || 6035 strcmp(argv[i], "-af") == 0 || 6036 strcmp(argv[i], "-vaf") == 0 || 6037 strcmp(argv[i], "-comp") == 0) 6038 i++; 6039 6040 if(!delete) { 6041 comp = read_super(fd, &sBlk, argv[source + 1]); 6042 if(comp == NULL) { 6043 ERROR("Failed to read existing filesystem - will not " 6044 "overwrite - ABORTING!\n"); 6045 ERROR("To force Mksquashfs to write to this block " 6046 "device or file use -noappend\n"); 6047 EXIT_MKSQUASHFS(); 6048 } 6049 6050 block_log = slog(block_size = sBlk.block_size); 6051 noI = SQUASHFS_UNCOMPRESSED_INODES(sBlk.flags); 6052 noD = SQUASHFS_UNCOMPRESSED_DATA(sBlk.flags); 6053 noF = SQUASHFS_UNCOMPRESSED_FRAGMENTS(sBlk.flags); 6054 noX = SQUASHFS_UNCOMPRESSED_XATTRS(sBlk.flags); 6055 no_fragments = SQUASHFS_NO_FRAGMENTS(sBlk.flags); 6056 always_use_fragments = SQUASHFS_ALWAYS_FRAGMENTS(sBlk.flags); 6057 duplicate_checking = SQUASHFS_DUPLICATES(sBlk.flags); 6058 exportable = SQUASHFS_EXPORTABLE(sBlk.flags); 6059 no_xattrs = SQUASHFS_NO_XATTRS(sBlk.flags); 6060 comp_opts = SQUASHFS_COMP_OPTS(sBlk.flags); 6061 } 6062 6063 initialise_threads(readq, fragq, bwriteq, fwriteq, delete, 6064 destination_file); 6065 6066 res = compressor_init(comp, &stream, SQUASHFS_METADATA_SIZE, 0); 6067 if(res) 6068 BAD_ERROR("compressor_init failed\n"); 6069 6070 if(delete) { 6071 int size; 6072 void *comp_data = compressor_dump_options(comp, block_size, 6073 &size); 6074 6075 printf("Creating %d.%d filesystem on %s, block size %d.\n", 6076 SQUASHFS_MAJOR, SQUASHFS_MINOR, argv[source + 1], block_size); 6077 6078 /* 6079 * store any compressor specific options after the superblock, 6080 * and set the COMP_OPT flag to show that the filesystem has 6081 * compressor specfic options 6082 */ 6083 if(comp_data) { 6084 unsigned short c_byte = size | SQUASHFS_COMPRESSED_BIT; 6085 6086 SQUASHFS_INSWAP_SHORTS(&c_byte, 1); 6087 write_destination(fd, sizeof(struct squashfs_super_block), 6088 sizeof(c_byte), &c_byte); 6089 write_destination(fd, sizeof(struct squashfs_super_block) + 6090 sizeof(c_byte), size, comp_data); 6091 bytes = sizeof(struct squashfs_super_block) + sizeof(c_byte) 6092 + size; 6093 comp_opts = TRUE; 6094 } else 6095 bytes = sizeof(struct squashfs_super_block); 6096 } else { 6097 unsigned int last_directory_block, inode_dir_offset, 6098 inode_dir_file_size, root_inode_size, 6099 inode_dir_start_block, uncompressed_data, 6100 compressed_data, inode_dir_inode_number, 6101 inode_dir_parent_inode; 6102 unsigned int root_inode_start = 6103 SQUASHFS_INODE_BLK(sBlk.root_inode), 6104 root_inode_offset = 6105 SQUASHFS_INODE_OFFSET(sBlk.root_inode); 6106 6107 if((bytes = read_filesystem(root_name, fd, &sBlk, &inode_table, 6108 &data_cache, &directory_table, 6109 &directory_data_cache, &last_directory_block, 6110 &inode_dir_offset, &inode_dir_file_size, 6111 &root_inode_size, &inode_dir_start_block, 6112 &file_count, &sym_count, &dev_count, &dir_count, 6113 &fifo_count, &sock_count, &total_bytes, 6114 &total_inode_bytes, &total_directory_bytes, 6115 &inode_dir_inode_number, 6116 &inode_dir_parent_inode, add_old_root_entry, 6117 &fragment_table, &inode_lookup_table)) == 0) { 6118 ERROR("Failed to read existing filesystem - will not " 6119 "overwrite - ABORTING!\n"); 6120 ERROR("To force Mksquashfs to write to this block " 6121 "device or file use -noappend\n"); 6122 EXIT_MKSQUASHFS(); 6123 } 6124 if((append_fragments = fragments = sBlk.fragments)) { 6125 fragment_table = realloc((char *) fragment_table, 6126 ((fragments + FRAG_SIZE - 1) & ~(FRAG_SIZE - 1)) 6127 * sizeof(struct squashfs_fragment_entry)); 6128 if(fragment_table == NULL) 6129 BAD_ERROR("Out of memory in save filesystem state\n"); 6130 } 6131 6132 printf("Appending to existing %d.%d filesystem on %s, block " 6133 "size %d\n", SQUASHFS_MAJOR, SQUASHFS_MINOR, argv[source + 1], 6134 block_size); 6135 printf("All -b, -noI, -noD, -noF, -noX, no-duplicates, no-fragments, " 6136 "-always-use-fragments,\n-exportable and -comp options " 6137 "ignored\n"); 6138 printf("\nIf appending is not wanted, please re-run with " 6139 "-noappend specified!\n\n"); 6140 6141 compressed_data = (inode_dir_offset + inode_dir_file_size) & 6142 ~(SQUASHFS_METADATA_SIZE - 1); 6143 uncompressed_data = (inode_dir_offset + inode_dir_file_size) & 6144 (SQUASHFS_METADATA_SIZE - 1); 6145 6146 /* save original filesystem state for restoring ... */ 6147 sfragments = fragments; 6148 sbytes = bytes; 6149 sinode_count = sBlk.inodes; 6150 scache_bytes = root_inode_offset + root_inode_size; 6151 sdirectory_cache_bytes = uncompressed_data; 6152 sdata_cache = malloc(scache_bytes); 6153 if(sdata_cache == NULL) 6154 BAD_ERROR("Out of memory in save filesystem state\n"); 6155 sdirectory_data_cache = malloc(sdirectory_cache_bytes); 6156 if(sdirectory_data_cache == NULL) 6157 BAD_ERROR("Out of memory in save filesystem state\n"); 6158 memcpy(sdata_cache, data_cache, scache_bytes); 6159 memcpy(sdirectory_data_cache, directory_data_cache + 6160 compressed_data, sdirectory_cache_bytes); 6161 sinode_bytes = root_inode_start; 6162 stotal_bytes = total_bytes; 6163 stotal_inode_bytes = total_inode_bytes; 6164 stotal_directory_bytes = total_directory_bytes + 6165 compressed_data; 6166 sfile_count = file_count; 6167 ssym_count = sym_count; 6168 sdev_count = dev_count; 6169 sdir_count = dir_count + 1; 6170 sfifo_count = fifo_count; 6171 ssock_count = sock_count; 6172 sdup_files = dup_files; 6173 sid_count = id_count; 6174 write_recovery_data(&sBlk); 6175 save_xattrs(); 6176 appending = TRUE; 6177 6178 /* 6179 * set the filesystem state up to be able to append to the 6180 * original filesystem. The filesystem state differs depending 6181 * on whether we're appending to the original root directory, or 6182 * if the original root directory becomes a sub-directory 6183 * (root-becomes specified on command line, here root_name != 6184 * NULL) 6185 */ 6186 inode_bytes = inode_size = root_inode_start; 6187 directory_size = last_directory_block; 6188 cache_size = root_inode_offset + root_inode_size; 6189 directory_cache_size = inode_dir_offset + inode_dir_file_size; 6190 if(root_name) { 6191 sdirectory_bytes = last_directory_block; 6192 sdirectory_compressed_bytes = 0; 6193 root_inode_number = inode_dir_parent_inode; 6194 inode_no = sBlk.inodes + 2; 6195 directory_bytes = last_directory_block; 6196 directory_cache_bytes = uncompressed_data; 6197 memmove(directory_data_cache, directory_data_cache + 6198 compressed_data, uncompressed_data); 6199 cache_bytes = root_inode_offset + root_inode_size; 6200 add_old_root_entry(root_name, sBlk.root_inode, 6201 inode_dir_inode_number, SQUASHFS_DIR_TYPE); 6202 total_directory_bytes += compressed_data; 6203 dir_count ++; 6204 } else { 6205 sdirectory_compressed_bytes = last_directory_block - 6206 inode_dir_start_block; 6207 sdirectory_compressed = 6208 malloc(sdirectory_compressed_bytes); 6209 if(sdirectory_compressed == NULL) 6210 BAD_ERROR("Out of memory in save filesystem " 6211 "state\n"); 6212 memcpy(sdirectory_compressed, directory_table + 6213 inode_dir_start_block, 6214 sdirectory_compressed_bytes); 6215 sdirectory_bytes = inode_dir_start_block; 6216 root_inode_number = inode_dir_inode_number; 6217 inode_no = sBlk.inodes + 1; 6218 directory_bytes = inode_dir_start_block; 6219 directory_cache_bytes = inode_dir_offset; 6220 cache_bytes = root_inode_offset; 6221 } 6222 6223 inode_count = file_count + dir_count + sym_count + dev_count + 6224 fifo_count + sock_count; 6225 } 6226 6227 if(path) 6228 paths = add_subdir(paths, path); 6229 6230 dump_actions(); 6231 dump_pseudos(); 6232 6233 if(delete && !keep_as_directory && source == 1 && 6234 S_ISDIR(source_buf.st_mode)) 6235 dir_scan(&inode, source_path[0], scan1_readdir, progress); 6236 else if(!keep_as_directory && source == 1 && 6237 S_ISDIR(source_buf.st_mode)) 6238 dir_scan(&inode, source_path[0], scan1_single_readdir, progress); 6239 else 6240 dir_scan(&inode, "", scan1_encomp_readdir, progress); 6241 sBlk.root_inode = inode; 6242 sBlk.inodes = inode_count; 6243 sBlk.s_magic = SQUASHFS_MAGIC; 6244 sBlk.s_major = SQUASHFS_MAJOR; 6245 sBlk.s_minor = SQUASHFS_MINOR; 6246 sBlk.block_size = block_size; 6247 sBlk.block_log = block_log; 6248 sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments, 6249 always_use_fragments, duplicate_checking, exportable, 6250 no_xattrs, comp_opts); 6251 sBlk.mkfs_time = time(NULL); 6252 6253 disable_info(); 6254 6255 while((fragment = get_frag_action(fragment))) 6256 write_fragment(*fragment); 6257 unlock_fragments(); 6258 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 6259 pthread_mutex_lock(&fragment_mutex); 6260 while(fragments_outstanding) { 6261 pthread_mutex_unlock(&fragment_mutex); 6262 sched_yield(); 6263 pthread_mutex_lock(&fragment_mutex); 6264 } 6265 pthread_cleanup_pop(1); 6266 6267 queue_put(to_writer, NULL); 6268 if(queue_get(from_writer) != 0) 6269 EXIT_MKSQUASHFS(); 6270 6271 set_progressbar_state(FALSE); 6272 write_filesystem_tables(&sBlk, nopad); 6273 6274 /* ANDROID CHANGES START*/ 6275 #ifdef ANDROID 6276 if (block_map_file) 6277 fclose(block_map_file); 6278 #endif 6279 /* ANDROID CHANGES END */ 6280 6281 return 0; 6282 } 6283
