1 /* 2 * Create a squashfs filesystem. This is a highly compressed read only 3 * filesystem. 4 * 5 * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 6 * 2012, 2013, 2014 7 * Phillip Lougher <phillip (at) squashfs.org.uk> 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 2, 12 * or (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software 21 * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 22 * 23 * mksquashfs.c 24 */ 25 26 #define FALSE 0 27 #define TRUE 1 28 #define MAX_LINE 16384 29 30 #include <pwd.h> 31 #include <grp.h> 32 #include <time.h> 33 #include <unistd.h> 34 #include <stdio.h> 35 #include <stddef.h> 36 #include <sys/types.h> 37 #include <sys/stat.h> 38 #include <fcntl.h> 39 #include <errno.h> 40 #include <dirent.h> 41 #include <string.h> 42 #include <stdlib.h> 43 #include <signal.h> 44 #include <setjmp.h> 45 #include <sys/types.h> 46 #include <sys/mman.h> 47 #include <pthread.h> 48 #include <regex.h> 49 #include <fnmatch.h> 50 #include <sys/wait.h> 51 #include <limits.h> 52 #include <ctype.h> 53 54 #ifndef FNM_EXTMATCH /* glibc extension */ 55 #define FNM_EXTMATCH 0 56 #endif 57 58 #ifndef linux 59 #define __BYTE_ORDER BYTE_ORDER 60 #define __BIG_ENDIAN BIG_ENDIAN 61 #define __LITTLE_ENDIAN LITTLE_ENDIAN 62 #include <sys/sysctl.h> 63 #else 64 #include <endian.h> 65 #include <sys/sysinfo.h> 66 #endif 67 68 #include "squashfs_fs.h" 69 #include "squashfs_swap.h" 70 #include "mksquashfs.h" 71 #include "sort.h" 72 #include "pseudo.h" 73 #include "compressor.h" 74 #include "xattr.h" 75 #include "action.h" 76 #include "error.h" 77 #include "progressbar.h" 78 #include "info.h" 79 #include "caches-queues-lists.h" 80 #include "read_fs.h" 81 #include "restore.h" 82 #include "process_fragments.h" 83 84 /* ANDROID CHANGES START*/ 85 #ifdef ANDROID 86 #include "android.h" 87 #include "private/android_filesystem_config.h" 88 #include "private/canned_fs_config.h" 89 int android_config = FALSE; 90 char *context_file = NULL; 91 char *mount_point = NULL; 92 char *target_out_path = NULL; 93 fs_config_func_t fs_config_func = NULL; 94 int compress_thresh_per = 0; 95 int align_4k_blocks = TRUE; 96 FILE *block_map_file = NULL; 97 #endif 98 /* ANDROID CHANGES END */ 99 100 int delete = FALSE; 101 int fd; 102 struct squashfs_super_block sBlk; 103 104 /* filesystem flags for building */ 105 int comp_opts = FALSE; 106 int no_xattrs = XATTR_DEF; 107 int noX = FALSE; 108 int duplicate_checking = TRUE; 109 int noF = FALSE; 110 int no_fragments = FALSE; 111 int always_use_fragments = FALSE; 112 int noI = FALSE; 113 int noD = FALSE; 114 int silent = TRUE; 115 int exportable = TRUE; 116 int sparse_files = TRUE; 117 int old_exclude = TRUE; 118 int use_regex = FALSE; 119 int nopad = FALSE; 120 int exit_on_error = FALSE; 121 122 long long global_uid = -1, global_gid = -1; 123 124 /* superblock attributes */ 125 int block_size = SQUASHFS_FILE_SIZE, block_log; 126 unsigned int id_count = 0; 127 int file_count = 0, sym_count = 0, dev_count = 0, dir_count = 0, fifo_count = 0, 128 sock_count = 0; 129 130 /* write position within data section */ 131 long long bytes = 0, total_bytes = 0; 132 133 /* in memory directory table - possibly compressed */ 134 char *directory_table = NULL; 135 unsigned int directory_bytes = 0, directory_size = 0, total_directory_bytes = 0; 136 137 /* cached directory table */ 138 char *directory_data_cache = NULL; 139 unsigned int directory_cache_bytes = 0, directory_cache_size = 0; 140 141 /* in memory inode table - possibly compressed */ 142 char *inode_table = NULL; 143 unsigned int inode_bytes = 0, inode_size = 0, total_inode_bytes = 0; 144 145 /* cached inode table */ 146 char *data_cache = NULL; 147 unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; 148 149 /* inode lookup table */ 150 squashfs_inode *inode_lookup_table = NULL; 151 152 /* in memory directory data */ 153 #define I_COUNT_SIZE 128 154 #define DIR_ENTRIES 32 155 #define INODE_HASH_SIZE 65536 156 #define INODE_HASH_MASK (INODE_HASH_SIZE - 1) 157 #define INODE_HASH(dev, ino) (ino & INODE_HASH_MASK) 158 159 struct cached_dir_index { 160 struct squashfs_dir_index index; 161 char *name; 162 }; 163 164 struct directory { 165 unsigned int start_block; 166 unsigned int size; 167 unsigned char *buff; 168 unsigned char *p; 169 unsigned int entry_count; 170 unsigned char *entry_count_p; 171 unsigned int i_count; 172 unsigned int i_size; 173 struct cached_dir_index *index; 174 unsigned char *index_count_p; 175 unsigned int inode_number; 176 }; 177 178 struct inode_info *inode_info[INODE_HASH_SIZE]; 179 180 /* hash tables used to do fast duplicate searches in duplicate check */ 181 struct file_info *dupl[65536]; 182 int dup_files = 0; 183 184 /* exclude file handling */ 185 /* list of exclude dirs/files */ 186 struct exclude_info { 187 dev_t st_dev; 188 ino_t st_ino; 189 }; 190 191 #define EXCLUDE_SIZE 8192 192 int exclude = 0; 193 struct exclude_info *exclude_paths = NULL; 194 int old_excluded(char *filename, struct stat *buf); 195 196 struct path_entry { 197 char *name; 198 regex_t *preg; 199 struct pathname *paths; 200 }; 201 202 struct pathname { 203 int names; 204 struct path_entry *name; 205 }; 206 207 struct pathnames { 208 int count; 209 struct pathname *path[0]; 210 }; 211 #define PATHS_ALLOC_SIZE 10 212 213 struct pathnames *paths = NULL; 214 struct pathname *path = NULL; 215 struct pathname *stickypath = NULL; 216 int excluded(char *name, struct pathnames *paths, struct pathnames **new); 217 218 int fragments = 0; 219 220 #define FRAG_SIZE 32768 221 222 struct squashfs_fragment_entry *fragment_table = NULL; 223 int fragments_outstanding = 0; 224 225 int fragments_locked = FALSE; 226 227 /* current inode number for directories and non directories */ 228 unsigned int inode_no = 1; 229 unsigned int root_inode_number = 0; 230 231 /* list of source dirs/files */ 232 int source = 0; 233 char **source_path; 234 235 /* list of root directory entries read from original filesystem */ 236 int old_root_entries = 0; 237 struct old_root_entry_info { 238 char *name; 239 struct inode_info inode; 240 }; 241 struct old_root_entry_info *old_root_entry; 242 243 /* restore orignal filesystem state if appending to existing filesystem is 244 * cancelled */ 245 int appending = FALSE; 246 char *sdata_cache, *sdirectory_data_cache, *sdirectory_compressed; 247 248 long long sbytes, stotal_bytes; 249 250 unsigned int sinode_bytes, scache_bytes, sdirectory_bytes, 251 sdirectory_cache_bytes, sdirectory_compressed_bytes, 252 stotal_inode_bytes, stotal_directory_bytes, 253 sinode_count = 0, sfile_count, ssym_count, sdev_count, 254 sdir_count, sfifo_count, ssock_count, sdup_files; 255 int sfragments; 256 int threads; 257 258 /* flag whether destination file is a block device */ 259 int block_device = FALSE; 260 261 /* flag indicating whether files are sorted using sort list(s) */ 262 int sorted = FALSE; 263 264 /* save destination file name for deleting on error */ 265 char *destination_file = NULL; 266 267 /* recovery file for abnormal exit on appending */ 268 char *recovery_file = NULL; 269 int recover = TRUE; 270 271 struct id *id_hash_table[ID_ENTRIES]; 272 struct id *id_table[SQUASHFS_IDS], *sid_table[SQUASHFS_IDS]; 273 unsigned int uid_count = 0, guid_count = 0; 274 unsigned int sid_count = 0, suid_count = 0, sguid_count = 0; 275 276 struct cache *reader_buffer, *fragment_buffer, *reserve_cache; 277 struct cache *bwriter_buffer, *fwriter_buffer; 278 struct queue *to_reader, *to_deflate, *to_writer, *from_writer, 279 *to_frag, *locked_fragment, *to_process_frag; 280 struct seq_queue *to_main; 281 pthread_t reader_thread, writer_thread, main_thread; 282 pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; 283 pthread_t *restore_thread = NULL; 284 pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER; 285 pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER; 286 pthread_mutex_t dup_mutex = PTHREAD_MUTEX_INITIALIZER; 287 288 /* user options that control parallelisation */ 289 int processors = -1; 290 int bwriter_size; 291 292 /* compression operations */ 293 struct compressor *comp = NULL; 294 int compressor_opt_parsed = FALSE; 295 void *stream = NULL; 296 297 /* xattr stats */ 298 unsigned int xattr_bytes = 0, total_xattr_bytes = 0; 299 300 /* fragment to file mapping used when appending */ 301 int append_fragments = 0; 302 struct append_file **file_mapping; 303 304 /* root of the in-core directory structure */ 305 struct dir_info *root_dir; 306 307 static char *read_from_disk(long long start, unsigned int avail_bytes); 308 void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, 309 int type); 310 struct file_info *duplicate(long long file_size, long long bytes, 311 unsigned int **block_list, long long *start, struct fragment **fragment, 312 struct file_buffer *file_buffer, int blocks, unsigned short checksum, 313 int checksum_flag); 314 struct dir_info *dir_scan1(char *, char *, struct pathnames *, 315 struct dir_ent *(_readdir)(struct dir_info *), int); 316 void dir_scan2(struct dir_info *dir, struct pseudo *pseudo); 317 void dir_scan3(struct dir_info *dir); 318 void dir_scan4(struct dir_info *dir); 319 void dir_scan5(struct dir_info *dir); 320 void dir_scan6(struct dir_info *dir); 321 void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info); 322 struct file_info *add_non_dup(long long file_size, long long bytes, 323 unsigned int *block_list, long long start, struct fragment *fragment, 324 unsigned short checksum, unsigned short fragment_checksum, 325 int checksum_flag, int checksum_frag_flag); 326 long long generic_write_table(int, void *, int, void *, int); 327 void restorefs(); 328 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth); 329 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad); 330 unsigned short get_checksum_mem(char *buff, int bytes); 331 void check_usable_phys_mem(int total_mem); 332 333 334 void prep_exit() 335 { 336 if(restore_thread) { 337 if(pthread_self() == *restore_thread) { 338 /* 339 * Recursive failure when trying to restore filesystem! 340 * Nothing to do except to exit, otherwise we'll just 341 * appear to hang. The user should be able to restore 342 * from the recovery file (which is why it was added, in 343 * case of catastrophic failure in Mksquashfs) 344 */ 345 exit(1); 346 } else { 347 /* signal the restore thread to restore */ 348 pthread_kill(*restore_thread, SIGUSR1); 349 pthread_exit(NULL); 350 } 351 } else if(delete) { 352 if(destination_file && !block_device) 353 unlink(destination_file); 354 } else if(recovery_file) 355 unlink(recovery_file); 356 } 357 358 359 int add_overflow(int a, int b) 360 { 361 return (INT_MAX - a) < b; 362 } 363 364 365 int shift_overflow(int a, int shift) 366 { 367 return (INT_MAX >> shift) < a; 368 } 369 370 371 int multiply_overflow(int a, int multiplier) 372 { 373 return (INT_MAX / multiplier) < a; 374 } 375 376 377 int multiply_overflowll(long long a, int multiplier) 378 { 379 return (LLONG_MAX / multiplier) < a; 380 } 381 382 383 #define MKINODE(A) ((squashfs_inode)(((squashfs_inode) inode_bytes << 16) \ 384 + (((char *)A) - data_cache))) 385 386 387 void restorefs() 388 { 389 ERROR("Exiting - restoring original filesystem!\n\n"); 390 391 bytes = sbytes; 392 memcpy(data_cache, sdata_cache, cache_bytes = scache_bytes); 393 memcpy(directory_data_cache, sdirectory_data_cache, 394 sdirectory_cache_bytes); 395 directory_cache_bytes = sdirectory_cache_bytes; 396 inode_bytes = sinode_bytes; 397 directory_bytes = sdirectory_bytes; 398 memcpy(directory_table + directory_bytes, sdirectory_compressed, 399 sdirectory_compressed_bytes); 400 directory_bytes += sdirectory_compressed_bytes; 401 total_bytes = stotal_bytes; 402 total_inode_bytes = stotal_inode_bytes; 403 total_directory_bytes = stotal_directory_bytes; 404 inode_count = sinode_count; 405 file_count = sfile_count; 406 sym_count = ssym_count; 407 dev_count = sdev_count; 408 dir_count = sdir_count; 409 fifo_count = sfifo_count; 410 sock_count = ssock_count; 411 dup_files = sdup_files; 412 fragments = sfragments; 413 id_count = sid_count; 414 restore_xattrs(); 415 write_filesystem_tables(&sBlk, nopad); 416 exit(1); 417 } 418 419 420 void sighandler() 421 { 422 EXIT_MKSQUASHFS(); 423 } 424 425 426 int mangle2(void *strm, char *d, char *s, int size, 427 int block_size, int uncompressed, int data_block) 428 { 429 int error, c_byte = 0; 430 431 if(!uncompressed) { 432 c_byte = compressor_compress(comp, strm, d, s, size, block_size, 433 &error); 434 if(c_byte == -1) 435 BAD_ERROR("mangle2:: %s compress failed with error " 436 "code %d\n", comp->name, error); 437 } 438 439 if(c_byte == 0 || c_byte >= size || 440 (c_byte > (size * ((100.0 - compress_thresh_per) / 100.0)))) { 441 memcpy(d, s, size); 442 return size | (data_block ? SQUASHFS_COMPRESSED_BIT_BLOCK : 443 SQUASHFS_COMPRESSED_BIT); 444 } 445 446 return c_byte; 447 } 448 449 450 int mangle(char *d, char *s, int size, int block_size, 451 int uncompressed, int data_block) 452 { 453 return mangle2(stream, d, s, size, block_size, uncompressed, 454 data_block); 455 } 456 457 458 void *get_inode(int req_size) 459 { 460 int data_space; 461 unsigned short c_byte; 462 463 while(cache_bytes >= SQUASHFS_METADATA_SIZE) { 464 if((inode_size - inode_bytes) < 465 ((SQUASHFS_METADATA_SIZE << 1)) + 2) { 466 void *it = realloc(inode_table, inode_size + 467 (SQUASHFS_METADATA_SIZE << 1) + 2); 468 if(it == NULL) 469 MEM_ERROR(); 470 inode_table = it; 471 inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 472 } 473 474 c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, 475 data_cache, SQUASHFS_METADATA_SIZE, 476 SQUASHFS_METADATA_SIZE, noI, 0); 477 TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); 478 SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); 479 inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; 480 total_inode_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; 481 memmove(data_cache, data_cache + SQUASHFS_METADATA_SIZE, 482 cache_bytes - SQUASHFS_METADATA_SIZE); 483 cache_bytes -= SQUASHFS_METADATA_SIZE; 484 } 485 486 data_space = (cache_size - cache_bytes); 487 if(data_space < req_size) { 488 int realloc_size = cache_size == 0 ? 489 ((req_size + SQUASHFS_METADATA_SIZE) & 490 ~(SQUASHFS_METADATA_SIZE - 1)) : req_size - 491 data_space; 492 493 void *dc = realloc(data_cache, cache_size + 494 realloc_size); 495 if(dc == NULL) 496 MEM_ERROR(); 497 cache_size += realloc_size; 498 data_cache = dc; 499 } 500 501 cache_bytes += req_size; 502 503 return data_cache + cache_bytes - req_size; 504 } 505 506 507 int read_bytes(int fd, void *buff, int bytes) 508 { 509 int res, count; 510 511 for(count = 0; count < bytes; count += res) { 512 res = read(fd, buff + count, bytes - count); 513 if(res < 1) { 514 if(res == 0) 515 goto bytes_read; 516 else if(errno != EINTR) { 517 ERROR("Read failed because %s\n", 518 strerror(errno)); 519 return -1; 520 } else 521 res = 0; 522 } 523 } 524 525 bytes_read: 526 return count; 527 } 528 529 530 int read_fs_bytes(int fd, long long byte, int bytes, void *buff) 531 { 532 off_t off = byte; 533 int res = 1; 534 535 TRACE("read_fs_bytes: reading from position 0x%llx, bytes %d\n", 536 byte, bytes); 537 538 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 539 pthread_mutex_lock(&pos_mutex); 540 if(lseek(fd, off, SEEK_SET) == -1) { 541 ERROR("read_fs_bytes: Lseek on destination failed because %s, " 542 "offset=0x%llx\n", strerror(errno), off); 543 res = 0; 544 } else if(read_bytes(fd, buff, bytes) < bytes) { 545 ERROR("Read on destination failed\n"); 546 res = 0; 547 } 548 549 pthread_cleanup_pop(1); 550 return res; 551 } 552 553 554 int write_bytes(int fd, void *buff, int bytes) 555 { 556 int res, count; 557 558 for(count = 0; count < bytes; count += res) { 559 res = write(fd, buff + count, bytes - count); 560 if(res == -1) { 561 if(errno != EINTR) { 562 ERROR("Write failed because %s\n", 563 strerror(errno)); 564 return -1; 565 } 566 res = 0; 567 } 568 } 569 570 return 0; 571 } 572 573 574 void write_destination(int fd, long long byte, int bytes, void *buff) 575 { 576 off_t off = byte; 577 578 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 579 pthread_mutex_lock(&pos_mutex); 580 581 if(lseek(fd, off, SEEK_SET) == -1) { 582 ERROR("write_destination: Lseek on destination " 583 "failed because %s, offset=0x%llx\n", strerror(errno), 584 off); 585 BAD_ERROR("Probably out of space on output %s\n", 586 block_device ? "block device" : "filesystem"); 587 } 588 589 if(write_bytes(fd, buff, bytes) == -1) 590 BAD_ERROR("Failed to write to output %s\n", 591 block_device ? "block device" : "filesystem"); 592 593 pthread_cleanup_pop(1); 594 } 595 596 597 long long write_inodes() 598 { 599 unsigned short c_byte; 600 int avail_bytes; 601 char *datap = data_cache; 602 long long start_bytes = bytes; 603 604 while(cache_bytes) { 605 if(inode_size - inode_bytes < 606 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 607 void *it = realloc(inode_table, inode_size + 608 ((SQUASHFS_METADATA_SIZE << 1) + 2)); 609 if(it == NULL) 610 MEM_ERROR(); 611 inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 612 inode_table = it; 613 } 614 avail_bytes = cache_bytes > SQUASHFS_METADATA_SIZE ? 615 SQUASHFS_METADATA_SIZE : cache_bytes; 616 c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, datap, 617 avail_bytes, SQUASHFS_METADATA_SIZE, noI, 0); 618 TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); 619 SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); 620 inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; 621 total_inode_bytes += avail_bytes + BLOCK_OFFSET; 622 datap += avail_bytes; 623 cache_bytes -= avail_bytes; 624 } 625 626 write_destination(fd, bytes, inode_bytes, inode_table); 627 bytes += inode_bytes; 628 629 return start_bytes; 630 } 631 632 633 long long write_directories() 634 { 635 unsigned short c_byte; 636 int avail_bytes; 637 char *directoryp = directory_data_cache; 638 long long start_bytes = bytes; 639 640 while(directory_cache_bytes) { 641 if(directory_size - directory_bytes < 642 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 643 void *dt = realloc(directory_table, 644 directory_size + ((SQUASHFS_METADATA_SIZE << 1) 645 + 2)); 646 if(dt == NULL) 647 MEM_ERROR(); 648 directory_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 649 directory_table = dt; 650 } 651 avail_bytes = directory_cache_bytes > SQUASHFS_METADATA_SIZE ? 652 SQUASHFS_METADATA_SIZE : directory_cache_bytes; 653 c_byte = mangle(directory_table + directory_bytes + 654 BLOCK_OFFSET, directoryp, avail_bytes, 655 SQUASHFS_METADATA_SIZE, noI, 0); 656 TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, 657 c_byte); 658 SQUASHFS_SWAP_SHORTS(&c_byte, 659 directory_table + directory_bytes, 1); 660 directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + 661 BLOCK_OFFSET; 662 total_directory_bytes += avail_bytes + BLOCK_OFFSET; 663 directoryp += avail_bytes; 664 directory_cache_bytes -= avail_bytes; 665 } 666 write_destination(fd, bytes, directory_bytes, directory_table); 667 bytes += directory_bytes; 668 669 return start_bytes; 670 } 671 672 673 long long write_id_table() 674 { 675 unsigned int id_bytes = SQUASHFS_ID_BYTES(id_count); 676 unsigned int p[id_count]; 677 int i; 678 679 TRACE("write_id_table: ids %d, id_bytes %d\n", id_count, id_bytes); 680 for(i = 0; i < id_count; i++) { 681 TRACE("write_id_table: id index %d, id %d", i, id_table[i]->id); 682 SQUASHFS_SWAP_INTS(&id_table[i]->id, p + i, 1); 683 } 684 685 return generic_write_table(id_bytes, p, 0, NULL, noI); 686 } 687 688 689 struct id *get_id(unsigned int id) 690 { 691 int hash = ID_HASH(id); 692 struct id *entry = id_hash_table[hash]; 693 694 for(; entry; entry = entry->next) 695 if(entry->id == id) 696 break; 697 698 return entry; 699 } 700 701 702 struct id *create_id(unsigned int id) 703 { 704 int hash = ID_HASH(id); 705 struct id *entry = malloc(sizeof(struct id)); 706 if(entry == NULL) 707 MEM_ERROR(); 708 entry->id = id; 709 entry->index = id_count ++; 710 entry->flags = 0; 711 entry->next = id_hash_table[hash]; 712 id_hash_table[hash] = entry; 713 id_table[entry->index] = entry; 714 return entry; 715 } 716 717 718 unsigned int get_uid(unsigned int uid) 719 { 720 struct id *entry = get_id(uid); 721 722 if(entry == NULL) { 723 if(id_count == SQUASHFS_IDS) 724 BAD_ERROR("Out of uids!\n"); 725 entry = create_id(uid); 726 } 727 728 if((entry->flags & ISA_UID) == 0) { 729 entry->flags |= ISA_UID; 730 uid_count ++; 731 } 732 733 return entry->index; 734 } 735 736 737 unsigned int get_guid(unsigned int guid) 738 { 739 struct id *entry = get_id(guid); 740 741 if(entry == NULL) { 742 if(id_count == SQUASHFS_IDS) 743 BAD_ERROR("Out of gids!\n"); 744 entry = create_id(guid); 745 } 746 747 if((entry->flags & ISA_GID) == 0) { 748 entry->flags |= ISA_GID; 749 guid_count ++; 750 } 751 752 return entry->index; 753 } 754 755 756 #define ALLOC_SIZE 128 757 758 char *_pathname(struct dir_ent *dir_ent, char *pathname, int *size) 759 { 760 if(pathname == NULL) { 761 pathname = malloc(ALLOC_SIZE); 762 if(pathname == NULL) 763 MEM_ERROR(); 764 } 765 766 for(;;) { 767 int res = snprintf(pathname, *size, "%s/%s", 768 dir_ent->our_dir->pathname, 769 dir_ent->source_name ? : dir_ent->name); 770 771 if(res < 0) 772 BAD_ERROR("snprintf failed in pathname\n"); 773 else if(res >= *size) { 774 /* 775 * pathname is too small to contain the result, so 776 * increase it and try again 777 */ 778 *size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); 779 pathname = realloc(pathname, *size); 780 if(pathname == NULL) 781 MEM_ERROR(); 782 } else 783 break; 784 } 785 786 return pathname; 787 } 788 789 790 char *pathname(struct dir_ent *dir_ent) 791 { 792 static char *pathname = NULL; 793 static int size = ALLOC_SIZE; 794 795 if (dir_ent->nonstandard_pathname) 796 return dir_ent->nonstandard_pathname; 797 798 return pathname = _pathname(dir_ent, pathname, &size); 799 } 800 801 802 char *pathname_reader(struct dir_ent *dir_ent) 803 { 804 static char *pathname = NULL; 805 static int size = ALLOC_SIZE; 806 807 if (dir_ent->nonstandard_pathname) 808 return dir_ent->nonstandard_pathname; 809 810 return pathname = _pathname(dir_ent, pathname, &size); 811 } 812 813 814 char *subpathname(struct dir_ent *dir_ent) 815 { 816 static char *subpath = NULL; 817 static int size = ALLOC_SIZE; 818 int res; 819 820 if(subpath == NULL) { 821 subpath = malloc(ALLOC_SIZE); 822 if(subpath == NULL) 823 MEM_ERROR(); 824 } 825 826 for(;;) { 827 if(dir_ent->our_dir->subpath[0] != '\0') 828 res = snprintf(subpath, size, "%s/%s", 829 dir_ent->our_dir->subpath, dir_ent->name); 830 else 831 res = snprintf(subpath, size, "/%s", dir_ent->name); 832 833 if(res < 0) 834 BAD_ERROR("snprintf failed in subpathname\n"); 835 else if(res >= size) { 836 /* 837 * subpath is too small to contain the result, so 838 * increase it and try again 839 */ 840 size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); 841 subpath = realloc(subpath, size); 842 if(subpath == NULL) 843 MEM_ERROR(); 844 } else 845 break; 846 } 847 848 return subpath; 849 } 850 851 852 static inline unsigned int get_inode_no(struct inode_info *inode) 853 { 854 return inode->inode_number; 855 } 856 857 858 static inline unsigned int get_parent_no(struct dir_info *dir) 859 { 860 return dir->depth ? get_inode_no(dir->dir_ent->inode) : inode_no; 861 } 862 863 864 /* ANDROID CHANGES START*/ 865 #ifdef ANDROID 866 static inline void write_block_map_entry(char *sub_path, unsigned long long start_block, unsigned long long total_size, 867 char * mount_point, FILE *block_map_file) { 868 if (block_map_file) { 869 unsigned long long round_start = (start_block + (1 << 12) - 1) >> 12; 870 unsigned long long round_end = ((start_block + total_size) >> 12) - 1; 871 if (round_start && total_size && round_start <= round_end) { 872 fprintf(block_map_file, "/%s", mount_point); 873 if (sub_path[0] != '/') fprintf(block_map_file, "/"); 874 if (round_start == round_end) 875 fprintf(block_map_file, "%s %lld\n", sub_path, round_start); 876 else 877 fprintf(block_map_file, "%s %lld-%lld\n", sub_path, round_start, round_end); 878 } 879 } 880 } 881 #endif 882 /* ANDROID CHANGES END */ 883 884 int create_inode(squashfs_inode *i_no, struct dir_info *dir_info, 885 struct dir_ent *dir_ent, int type, long long byte_size, 886 long long start_block, unsigned int offset, unsigned int *block_list, 887 struct fragment *fragment, struct directory *dir_in, long long sparse) 888 { 889 struct stat *buf = &dir_ent->inode->buf; 890 union squashfs_inode_header inode_header; 891 struct squashfs_base_inode_header *base = &inode_header.base; 892 void *inode; 893 char *filename = pathname(dir_ent); 894 int nlink = dir_ent->inode->nlink; 895 int xattr = read_xattrs(dir_ent); 896 897 switch(type) { 898 case SQUASHFS_FILE_TYPE: 899 if(dir_ent->inode->nlink > 1 || 900 byte_size >= (1LL << 32) || 901 start_block >= (1LL << 32) || 902 sparse || IS_XATTR(xattr)) 903 type = SQUASHFS_LREG_TYPE; 904 break; 905 case SQUASHFS_DIR_TYPE: 906 if(dir_info->dir_is_ldir || IS_XATTR(xattr)) 907 type = SQUASHFS_LDIR_TYPE; 908 break; 909 case SQUASHFS_SYMLINK_TYPE: 910 if(IS_XATTR(xattr)) 911 type = SQUASHFS_LSYMLINK_TYPE; 912 break; 913 case SQUASHFS_BLKDEV_TYPE: 914 if(IS_XATTR(xattr)) 915 type = SQUASHFS_LBLKDEV_TYPE; 916 break; 917 case SQUASHFS_CHRDEV_TYPE: 918 if(IS_XATTR(xattr)) 919 type = SQUASHFS_LCHRDEV_TYPE; 920 break; 921 case SQUASHFS_FIFO_TYPE: 922 if(IS_XATTR(xattr)) 923 type = SQUASHFS_LFIFO_TYPE; 924 break; 925 case SQUASHFS_SOCKET_TYPE: 926 if(IS_XATTR(xattr)) 927 type = SQUASHFS_LSOCKET_TYPE; 928 break; 929 } 930 931 base->mode = SQUASHFS_MODE(buf->st_mode); 932 base->uid = get_uid((unsigned int) global_uid == -1 ? 933 buf->st_uid : global_uid); 934 base->inode_type = type; 935 base->guid = get_guid((unsigned int) global_gid == -1 ? 936 buf->st_gid : global_gid); 937 base->mtime = buf->st_mtime; 938 base->inode_number = get_inode_no(dir_ent->inode); 939 940 if(type == SQUASHFS_FILE_TYPE) { 941 int i; 942 struct squashfs_reg_inode_header *reg = &inode_header.reg; 943 size_t off = offsetof(struct squashfs_reg_inode_header, block_list); 944 /* ANDROID CHANGES START*/ 945 #ifdef ANDROID 946 unsigned long long total_size = 0; 947 char *sub_path; 948 #endif 949 /* ANDROID CHANGES END */ 950 951 inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); 952 reg->file_size = byte_size; 953 reg->start_block = start_block; 954 reg->fragment = fragment->index; 955 reg->offset = fragment->offset; 956 SQUASHFS_SWAP_REG_INODE_HEADER(reg, inode); 957 SQUASHFS_SWAP_INTS(block_list, inode + off, offset); 958 TRACE("File inode, file_size %lld, start_block 0x%llx, blocks " 959 "%d, fragment %d, offset %d, size %d\n", byte_size, 960 start_block, offset, fragment->index, fragment->offset, 961 fragment->size); 962 for(i = 0; i < offset; i++) { 963 TRACE("Block %d, size %d\n", i, block_list[i]); 964 total_size += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]); 965 } 966 /* ANDROID CHANGES START*/ 967 #ifdef ANDROID 968 sub_path = subpathname(dir_ent); 969 if (block_map_file && fragment->index == -1) { 970 write_block_map_entry(sub_path, start_block, total_size, mount_point, block_map_file); 971 } 972 #endif 973 /* ANDROID CHANGES END */ 974 } 975 else if(type == SQUASHFS_LREG_TYPE) { 976 /* ANDROID CHANGES START*/ 977 #ifdef ANDROID 978 unsigned long long total_size = 0; 979 char *sub_path; 980 #endif 981 /* ANDROID CHANGES END */ 982 int i; 983 struct squashfs_lreg_inode_header *reg = &inode_header.lreg; 984 size_t off = offsetof(struct squashfs_lreg_inode_header, block_list); 985 986 inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); 987 reg->nlink = nlink; 988 reg->file_size = byte_size; 989 reg->start_block = start_block; 990 reg->fragment = fragment->index; 991 reg->offset = fragment->offset; 992 if(sparse && sparse >= byte_size) 993 sparse = byte_size - 1; 994 reg->sparse = sparse; 995 reg->xattr = xattr; 996 SQUASHFS_SWAP_LREG_INODE_HEADER(reg, inode); 997 SQUASHFS_SWAP_INTS(block_list, inode + off, offset); 998 TRACE("Long file inode, file_size %lld, start_block 0x%llx, " 999 "blocks %d, fragment %d, offset %d, size %d, nlink %d" 1000 "\n", byte_size, start_block, offset, fragment->index, 1001 fragment->offset, fragment->size, nlink); 1002 for(i = 0; i < offset; i++) { 1003 TRACE("Block %d, size %d\n", i, block_list[i]); 1004 total_size += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]); 1005 } 1006 /* ANDROID CHANGES START*/ 1007 #ifdef ANDROID 1008 sub_path = subpathname(dir_ent); 1009 if (block_map_file && fragment->index == -1) { 1010 write_block_map_entry(sub_path, start_block, total_size, mount_point, block_map_file); 1011 } 1012 #endif 1013 /* ANDROID CHANGES END */ 1014 } 1015 else if(type == SQUASHFS_LDIR_TYPE) { 1016 int i; 1017 unsigned char *p; 1018 struct squashfs_ldir_inode_header *dir = &inode_header.ldir; 1019 struct cached_dir_index *index = dir_in->index; 1020 unsigned int i_count = dir_in->i_count; 1021 unsigned int i_size = dir_in->i_size; 1022 1023 if(byte_size >= 1 << 27) 1024 BAD_ERROR("directory greater than 2^27-1 bytes!\n"); 1025 1026 inode = get_inode(sizeof(*dir) + i_size); 1027 dir->inode_type = SQUASHFS_LDIR_TYPE; 1028 dir->nlink = dir_ent->dir->directory_count + 2; 1029 dir->file_size = byte_size; 1030 dir->offset = offset; 1031 dir->start_block = start_block; 1032 dir->i_count = i_count; 1033 dir->parent_inode = get_parent_no(dir_ent->our_dir); 1034 dir->xattr = xattr; 1035 1036 SQUASHFS_SWAP_LDIR_INODE_HEADER(dir, inode); 1037 p = inode + offsetof(struct squashfs_ldir_inode_header, index); 1038 for(i = 0; i < i_count; i++) { 1039 SQUASHFS_SWAP_DIR_INDEX(&index[i].index, p); 1040 p += offsetof(struct squashfs_dir_index, name); 1041 memcpy(p, index[i].name, index[i].index.size + 1); 1042 p += index[i].index.size + 1; 1043 } 1044 TRACE("Long directory inode, file_size %lld, start_block " 1045 "0x%llx, offset 0x%x, nlink %d\n", byte_size, 1046 start_block, offset, dir_ent->dir->directory_count + 2); 1047 } 1048 else if(type == SQUASHFS_DIR_TYPE) { 1049 struct squashfs_dir_inode_header *dir = &inode_header.dir; 1050 1051 inode = get_inode(sizeof(*dir)); 1052 dir->nlink = dir_ent->dir->directory_count + 2; 1053 dir->file_size = byte_size; 1054 dir->offset = offset; 1055 dir->start_block = start_block; 1056 dir->parent_inode = get_parent_no(dir_ent->our_dir); 1057 SQUASHFS_SWAP_DIR_INODE_HEADER(dir, inode); 1058 TRACE("Directory inode, file_size %lld, start_block 0x%llx, " 1059 "offset 0x%x, nlink %d\n", byte_size, start_block, 1060 offset, dir_ent->dir->directory_count + 2); 1061 } 1062 else if(type == SQUASHFS_CHRDEV_TYPE || type == SQUASHFS_BLKDEV_TYPE) { 1063 struct squashfs_dev_inode_header *dev = &inode_header.dev; 1064 unsigned int major = major(buf->st_rdev); 1065 unsigned int minor = minor(buf->st_rdev); 1066 1067 if(major > 0xfff) { 1068 ERROR("Major %d out of range in device node %s, " 1069 "truncating to %d\n", major, filename, 1070 major & 0xfff); 1071 major &= 0xfff; 1072 } 1073 if(minor > 0xfffff) { 1074 ERROR("Minor %d out of range in device node %s, " 1075 "truncating to %d\n", minor, filename, 1076 minor & 0xfffff); 1077 minor &= 0xfffff; 1078 } 1079 inode = get_inode(sizeof(*dev)); 1080 dev->nlink = nlink; 1081 dev->rdev = (major << 8) | (minor & 0xff) | 1082 ((minor & ~0xff) << 12); 1083 SQUASHFS_SWAP_DEV_INODE_HEADER(dev, inode); 1084 TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); 1085 } 1086 else if(type == SQUASHFS_LCHRDEV_TYPE || type == SQUASHFS_LBLKDEV_TYPE) { 1087 struct squashfs_ldev_inode_header *dev = &inode_header.ldev; 1088 unsigned int major = major(buf->st_rdev); 1089 unsigned int minor = minor(buf->st_rdev); 1090 1091 if(major > 0xfff) { 1092 ERROR("Major %d out of range in device node %s, " 1093 "truncating to %d\n", major, filename, 1094 major & 0xfff); 1095 major &= 0xfff; 1096 } 1097 if(minor > 0xfffff) { 1098 ERROR("Minor %d out of range in device node %s, " 1099 "truncating to %d\n", minor, filename, 1100 minor & 0xfffff); 1101 minor &= 0xfffff; 1102 } 1103 inode = get_inode(sizeof(*dev)); 1104 dev->nlink = nlink; 1105 dev->rdev = (major << 8) | (minor & 0xff) | 1106 ((minor & ~0xff) << 12); 1107 dev->xattr = xattr; 1108 SQUASHFS_SWAP_LDEV_INODE_HEADER(dev, inode); 1109 TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); 1110 } 1111 else if(type == SQUASHFS_SYMLINK_TYPE) { 1112 struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; 1113 int byte = strlen(dir_ent->inode->symlink); 1114 size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); 1115 1116 inode = get_inode(sizeof(*symlink) + byte); 1117 symlink->nlink = nlink; 1118 symlink->symlink_size = byte; 1119 SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); 1120 strncpy(inode + off, dir_ent->inode->symlink, byte); 1121 TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, 1122 nlink); 1123 } 1124 else if(type == SQUASHFS_LSYMLINK_TYPE) { 1125 struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; 1126 int byte = strlen(dir_ent->inode->symlink); 1127 size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); 1128 1129 inode = get_inode(sizeof(*symlink) + byte + 1130 sizeof(unsigned int)); 1131 symlink->nlink = nlink; 1132 symlink->symlink_size = byte; 1133 SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); 1134 strncpy(inode + off, dir_ent->inode->symlink, byte); 1135 SQUASHFS_SWAP_INTS(&xattr, inode + off + byte, 1); 1136 TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, 1137 nlink); 1138 } 1139 else if(type == SQUASHFS_FIFO_TYPE || type == SQUASHFS_SOCKET_TYPE) { 1140 struct squashfs_ipc_inode_header *ipc = &inode_header.ipc; 1141 1142 inode = get_inode(sizeof(*ipc)); 1143 ipc->nlink = nlink; 1144 SQUASHFS_SWAP_IPC_INODE_HEADER(ipc, inode); 1145 TRACE("ipc inode, type %s, nlink %d\n", type == 1146 SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); 1147 } 1148 else if(type == SQUASHFS_LFIFO_TYPE || type == SQUASHFS_LSOCKET_TYPE) { 1149 struct squashfs_lipc_inode_header *ipc = &inode_header.lipc; 1150 1151 inode = get_inode(sizeof(*ipc)); 1152 ipc->nlink = nlink; 1153 ipc->xattr = xattr; 1154 SQUASHFS_SWAP_LIPC_INODE_HEADER(ipc, inode); 1155 TRACE("ipc inode, type %s, nlink %d\n", type == 1156 SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); 1157 } else 1158 BAD_ERROR("Unrecognised inode %d in create_inode\n", type); 1159 1160 *i_no = MKINODE(inode); 1161 inode_count ++; 1162 1163 TRACE("Created inode 0x%llx, type %d, uid %d, guid %d\n", *i_no, type, 1164 base->uid, base->guid); 1165 1166 return TRUE; 1167 } 1168 1169 1170 void add_dir(squashfs_inode inode, unsigned int inode_number, char *name, 1171 int type, struct directory *dir) 1172 { 1173 unsigned char *buff; 1174 struct squashfs_dir_entry idir; 1175 unsigned int start_block = inode >> 16; 1176 unsigned int offset = inode & 0xffff; 1177 unsigned int size = strlen(name); 1178 size_t name_off = offsetof(struct squashfs_dir_entry, name); 1179 1180 if(size > SQUASHFS_NAME_LEN) { 1181 size = SQUASHFS_NAME_LEN; 1182 ERROR("Filename is greater than %d characters, truncating! ..." 1183 "\n", SQUASHFS_NAME_LEN); 1184 } 1185 1186 if(dir->p + sizeof(struct squashfs_dir_entry) + size + 1187 sizeof(struct squashfs_dir_header) 1188 >= dir->buff + dir->size) { 1189 buff = realloc(dir->buff, dir->size += SQUASHFS_METADATA_SIZE); 1190 if(buff == NULL) 1191 MEM_ERROR(); 1192 1193 dir->p = (dir->p - dir->buff) + buff; 1194 if(dir->entry_count_p) 1195 dir->entry_count_p = (dir->entry_count_p - dir->buff + 1196 buff); 1197 dir->index_count_p = dir->index_count_p - dir->buff + buff; 1198 dir->buff = buff; 1199 } 1200 1201 if(dir->entry_count == 256 || start_block != dir->start_block || 1202 ((dir->entry_count_p != NULL) && 1203 ((dir->p + sizeof(struct squashfs_dir_entry) + size - 1204 dir->index_count_p) > SQUASHFS_METADATA_SIZE)) || 1205 ((long long) inode_number - dir->inode_number) > 32767 1206 || ((long long) inode_number - dir->inode_number) 1207 < -32768) { 1208 if(dir->entry_count_p) { 1209 struct squashfs_dir_header dir_header; 1210 1211 if((dir->p + sizeof(struct squashfs_dir_entry) + size - 1212 dir->index_count_p) > 1213 SQUASHFS_METADATA_SIZE) { 1214 if(dir->i_count % I_COUNT_SIZE == 0) { 1215 dir->index = realloc(dir->index, 1216 (dir->i_count + I_COUNT_SIZE) * 1217 sizeof(struct cached_dir_index)); 1218 if(dir->index == NULL) 1219 MEM_ERROR(); 1220 } 1221 dir->index[dir->i_count].index.index = 1222 dir->p - dir->buff; 1223 dir->index[dir->i_count].index.size = size - 1; 1224 dir->index[dir->i_count++].name = name; 1225 dir->i_size += sizeof(struct squashfs_dir_index) 1226 + size; 1227 dir->index_count_p = dir->p; 1228 } 1229 1230 dir_header.count = dir->entry_count - 1; 1231 dir_header.start_block = dir->start_block; 1232 dir_header.inode_number = dir->inode_number; 1233 SQUASHFS_SWAP_DIR_HEADER(&dir_header, 1234 dir->entry_count_p); 1235 1236 } 1237 1238 1239 dir->entry_count_p = dir->p; 1240 dir->start_block = start_block; 1241 dir->entry_count = 0; 1242 dir->inode_number = inode_number; 1243 dir->p += sizeof(struct squashfs_dir_header); 1244 } 1245 1246 idir.offset = offset; 1247 idir.type = type; 1248 idir.size = size - 1; 1249 idir.inode_number = ((long long) inode_number - dir->inode_number); 1250 SQUASHFS_SWAP_DIR_ENTRY(&idir, dir->p); 1251 strncpy((char *) dir->p + name_off, name, size); 1252 dir->p += sizeof(struct squashfs_dir_entry) + size; 1253 dir->entry_count ++; 1254 } 1255 1256 1257 void write_dir(squashfs_inode *inode, struct dir_info *dir_info, 1258 struct directory *dir) 1259 { 1260 unsigned int dir_size = dir->p - dir->buff; 1261 int data_space = directory_cache_size - directory_cache_bytes; 1262 unsigned int directory_block, directory_offset, i_count, index; 1263 unsigned short c_byte; 1264 1265 if(data_space < dir_size) { 1266 int realloc_size = directory_cache_size == 0 ? 1267 ((dir_size + SQUASHFS_METADATA_SIZE) & 1268 ~(SQUASHFS_METADATA_SIZE - 1)) : dir_size - data_space; 1269 1270 void *dc = realloc(directory_data_cache, 1271 directory_cache_size + realloc_size); 1272 if(dc == NULL) 1273 MEM_ERROR(); 1274 directory_cache_size += realloc_size; 1275 directory_data_cache = dc; 1276 } 1277 1278 if(dir_size) { 1279 struct squashfs_dir_header dir_header; 1280 1281 dir_header.count = dir->entry_count - 1; 1282 dir_header.start_block = dir->start_block; 1283 dir_header.inode_number = dir->inode_number; 1284 SQUASHFS_SWAP_DIR_HEADER(&dir_header, dir->entry_count_p); 1285 memcpy(directory_data_cache + directory_cache_bytes, dir->buff, 1286 dir_size); 1287 } 1288 directory_offset = directory_cache_bytes; 1289 directory_block = directory_bytes; 1290 directory_cache_bytes += dir_size; 1291 i_count = 0; 1292 index = SQUASHFS_METADATA_SIZE - directory_offset; 1293 1294 while(1) { 1295 while(i_count < dir->i_count && 1296 dir->index[i_count].index.index < index) 1297 dir->index[i_count++].index.start_block = 1298 directory_bytes; 1299 index += SQUASHFS_METADATA_SIZE; 1300 1301 if(directory_cache_bytes < SQUASHFS_METADATA_SIZE) 1302 break; 1303 1304 if((directory_size - directory_bytes) < 1305 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 1306 void *dt = realloc(directory_table, 1307 directory_size + (SQUASHFS_METADATA_SIZE << 1) 1308 + 2); 1309 if(dt == NULL) 1310 MEM_ERROR(); 1311 directory_size += SQUASHFS_METADATA_SIZE << 1; 1312 directory_table = dt; 1313 } 1314 1315 c_byte = mangle(directory_table + directory_bytes + 1316 BLOCK_OFFSET, directory_data_cache, 1317 SQUASHFS_METADATA_SIZE, SQUASHFS_METADATA_SIZE, 1318 noI, 0); 1319 TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, 1320 c_byte); 1321 SQUASHFS_SWAP_SHORTS(&c_byte, 1322 directory_table + directory_bytes, 1); 1323 directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + 1324 BLOCK_OFFSET; 1325 total_directory_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; 1326 memmove(directory_data_cache, directory_data_cache + 1327 SQUASHFS_METADATA_SIZE, directory_cache_bytes - 1328 SQUASHFS_METADATA_SIZE); 1329 directory_cache_bytes -= SQUASHFS_METADATA_SIZE; 1330 } 1331 1332 create_inode(inode, dir_info, dir_info->dir_ent, SQUASHFS_DIR_TYPE, 1333 dir_size + 3, directory_block, directory_offset, NULL, NULL, 1334 dir, 0); 1335 1336 #ifdef SQUASHFS_TRACE 1337 { 1338 unsigned char *dirp; 1339 int count; 1340 1341 TRACE("Directory contents of inode 0x%llx\n", *inode); 1342 dirp = dir->buff; 1343 while(dirp < dir->p) { 1344 char buffer[SQUASHFS_NAME_LEN + 1]; 1345 struct squashfs_dir_entry idir, *idirp; 1346 struct squashfs_dir_header dirh; 1347 SQUASHFS_SWAP_DIR_HEADER((struct squashfs_dir_header *) dirp, 1348 &dirh); 1349 count = dirh.count + 1; 1350 dirp += sizeof(struct squashfs_dir_header); 1351 1352 TRACE("\tStart block 0x%x, count %d\n", 1353 dirh.start_block, count); 1354 1355 while(count--) { 1356 idirp = (struct squashfs_dir_entry *) dirp; 1357 SQUASHFS_SWAP_DIR_ENTRY(idirp, &idir); 1358 strncpy(buffer, idirp->name, idir.size + 1); 1359 buffer[idir.size + 1] = '\0'; 1360 TRACE("\t\tname %s, inode offset 0x%x, type " 1361 "%d\n", buffer, idir.offset, idir.type); 1362 dirp += sizeof(struct squashfs_dir_entry) + idir.size + 1363 1; 1364 } 1365 } 1366 } 1367 #endif 1368 dir_count ++; 1369 } 1370 1371 1372 static struct file_buffer *get_fragment(struct fragment *fragment) 1373 { 1374 struct squashfs_fragment_entry *disk_fragment; 1375 struct file_buffer *buffer, *compressed_buffer; 1376 long long start_block; 1377 int res, size, index = fragment->index; 1378 char locked; 1379 1380 /* 1381 * Lookup fragment block in cache. 1382 * If the fragment block doesn't exist, then get the compressed version 1383 * from the writer cache or off disk, and decompress it. 1384 * 1385 * This routine has two things which complicate the code: 1386 * 1387 * 1. Multiple threads can simultaneously lookup/create the 1388 * same buffer. This means a buffer needs to be "locked" 1389 * when it is being filled in, to prevent other threads from 1390 * using it when it is not ready. This is because we now do 1391 * fragment duplicate checking in parallel. 1392 * 2. We have two caches which need to be checked for the 1393 * presence of fragment blocks: the normal fragment cache 1394 * and a "reserve" cache. The reserve cache is used to 1395 * prevent an unnecessary pipeline stall when the fragment cache 1396 * is full of fragments waiting to be compressed. 1397 */ 1398 1399 if(fragment->index == SQUASHFS_INVALID_FRAG) 1400 return NULL; 1401 1402 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1403 pthread_mutex_lock(&dup_mutex); 1404 1405 again: 1406 buffer = cache_lookup_nowait(fragment_buffer, index, &locked); 1407 if(buffer) { 1408 pthread_mutex_unlock(&dup_mutex); 1409 if(locked) 1410 /* got a buffer being filled in. Wait for it */ 1411 cache_wait_unlock(buffer); 1412 goto finished; 1413 } 1414 1415 /* not in fragment cache, is it in the reserve cache? */ 1416 buffer = cache_lookup_nowait(reserve_cache, index, &locked); 1417 if(buffer) { 1418 pthread_mutex_unlock(&dup_mutex); 1419 if(locked) 1420 /* got a buffer being filled in. Wait for it */ 1421 cache_wait_unlock(buffer); 1422 goto finished; 1423 } 1424 1425 /* in neither cache, try to get it from the fragment cache */ 1426 buffer = cache_get_nowait(fragment_buffer, index); 1427 if(!buffer) { 1428 /* 1429 * no room, get it from the reserve cache, this is 1430 * dimensioned so it will always have space (no more than 1431 * processors + 1 can have an outstanding reserve buffer) 1432 */ 1433 buffer = cache_get_nowait(reserve_cache, index); 1434 if(!buffer) { 1435 /* failsafe */ 1436 ERROR("no space in reserve cache\n"); 1437 goto again; 1438 } 1439 } 1440 1441 pthread_mutex_unlock(&dup_mutex); 1442 1443 compressed_buffer = cache_lookup(fwriter_buffer, index); 1444 1445 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1446 pthread_mutex_lock(&fragment_mutex); 1447 disk_fragment = &fragment_table[index]; 1448 size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size); 1449 start_block = disk_fragment->start_block; 1450 pthread_cleanup_pop(1); 1451 1452 if(SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) { 1453 int error; 1454 char *data; 1455 1456 if(compressed_buffer) 1457 data = compressed_buffer->data; 1458 else { 1459 data = read_from_disk(start_block, size); 1460 if(data == NULL) { 1461 ERROR("Failed to read fragment from output" 1462 " filesystem\n"); 1463 BAD_ERROR("Output filesystem corrupted?\n"); 1464 } 1465 } 1466 1467 res = compressor_uncompress(comp, buffer->data, data, size, 1468 block_size, &error); 1469 if(res == -1) 1470 BAD_ERROR("%s uncompress failed with error code %d\n", 1471 comp->name, error); 1472 } else if(compressed_buffer) 1473 memcpy(buffer->data, compressed_buffer->data, size); 1474 else { 1475 res = read_fs_bytes(fd, start_block, size, buffer->data); 1476 if(res == 0) { 1477 ERROR("Failed to read fragment from output " 1478 "filesystem\n"); 1479 BAD_ERROR("Output filesystem corrupted?\n"); 1480 } 1481 } 1482 1483 cache_unlock(buffer); 1484 cache_block_put(compressed_buffer); 1485 1486 finished: 1487 pthread_cleanup_pop(0); 1488 1489 return buffer; 1490 } 1491 1492 1493 unsigned short get_fragment_checksum(struct file_info *file) 1494 { 1495 struct file_buffer *frag_buffer; 1496 struct append_file *append; 1497 int res, index = file->fragment->index; 1498 unsigned short checksum; 1499 1500 if(index == SQUASHFS_INVALID_FRAG) 1501 return 0; 1502 1503 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1504 pthread_mutex_lock(&dup_mutex); 1505 res = file->have_frag_checksum; 1506 checksum = file->fragment_checksum; 1507 pthread_cleanup_pop(1); 1508 1509 if(res) 1510 return checksum; 1511 1512 frag_buffer = get_fragment(file->fragment); 1513 1514 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1515 1516 for(append = file_mapping[index]; append; append = append->next) { 1517 int offset = append->file->fragment->offset; 1518 int size = append->file->fragment->size; 1519 unsigned short cksum = 1520 get_checksum_mem(frag_buffer->data + offset, size); 1521 1522 if(file == append->file) 1523 checksum = cksum; 1524 1525 pthread_mutex_lock(&dup_mutex); 1526 append->file->fragment_checksum = cksum; 1527 append->file->have_frag_checksum = TRUE; 1528 pthread_mutex_unlock(&dup_mutex); 1529 } 1530 1531 cache_block_put(frag_buffer); 1532 pthread_cleanup_pop(0); 1533 1534 return checksum; 1535 } 1536 1537 1538 void lock_fragments() 1539 { 1540 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1541 pthread_mutex_lock(&fragment_mutex); 1542 fragments_locked = TRUE; 1543 pthread_cleanup_pop(1); 1544 } 1545 1546 1547 void unlock_fragments() 1548 { 1549 int frg, size; 1550 struct file_buffer *write_buffer; 1551 1552 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1553 pthread_mutex_lock(&fragment_mutex); 1554 1555 /* 1556 * Note queue_empty() is inherently racy with respect to concurrent 1557 * queue get and pushes. We avoid this because we're holding the 1558 * fragment_mutex which ensures no other threads can be using the 1559 * queue at this time. 1560 */ 1561 while(!queue_empty(locked_fragment)) { 1562 write_buffer = queue_get(locked_fragment); 1563 frg = write_buffer->block; 1564 size = SQUASHFS_COMPRESSED_SIZE_BLOCK(fragment_table[frg].size); 1565 fragment_table[frg].start_block = bytes; 1566 write_buffer->block = bytes; 1567 bytes += size; 1568 fragments_outstanding --; 1569 queue_put(to_writer, write_buffer); 1570 TRACE("fragment_locked writing fragment %d, compressed size %d" 1571 "\n", frg, size); 1572 } 1573 fragments_locked = FALSE; 1574 pthread_cleanup_pop(1); 1575 } 1576 1577 /* Called with the fragment_mutex locked */ 1578 void add_pending_fragment(struct file_buffer *write_buffer, int c_byte, 1579 int fragment) 1580 { 1581 fragment_table[fragment].size = c_byte; 1582 write_buffer->block = fragment; 1583 1584 queue_put(locked_fragment, write_buffer); 1585 } 1586 1587 1588 void write_fragment(struct file_buffer *fragment) 1589 { 1590 if(fragment == NULL) 1591 return; 1592 1593 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1594 pthread_mutex_lock(&fragment_mutex); 1595 fragment_table[fragment->block].unused = 0; 1596 fragments_outstanding ++; 1597 queue_put(to_frag, fragment); 1598 pthread_cleanup_pop(1); 1599 } 1600 1601 1602 struct file_buffer *allocate_fragment() 1603 { 1604 struct file_buffer *fragment = cache_get(fragment_buffer, fragments); 1605 1606 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1607 pthread_mutex_lock(&fragment_mutex); 1608 1609 if(fragments % FRAG_SIZE == 0) { 1610 void *ft = realloc(fragment_table, (fragments + 1611 FRAG_SIZE) * sizeof(struct squashfs_fragment_entry)); 1612 if(ft == NULL) 1613 MEM_ERROR(); 1614 fragment_table = ft; 1615 } 1616 1617 fragment->size = 0; 1618 fragment->block = fragments ++; 1619 1620 pthread_cleanup_pop(1); 1621 1622 return fragment; 1623 } 1624 1625 1626 static struct fragment empty_fragment = {SQUASHFS_INVALID_FRAG, 0, 0}; 1627 1628 1629 void free_fragment(struct fragment *fragment) 1630 { 1631 if(fragment != &empty_fragment) 1632 free(fragment); 1633 } 1634 1635 1636 struct fragment *get_and_fill_fragment(struct file_buffer *file_buffer, 1637 struct dir_ent *dir_ent) 1638 { 1639 struct fragment *ffrg; 1640 struct file_buffer **fragment; 1641 1642 if(file_buffer == NULL || file_buffer->size == 0) 1643 return &empty_fragment; 1644 1645 fragment = eval_frag_actions(root_dir, dir_ent); 1646 1647 if((*fragment) && (*fragment)->size + file_buffer->size > block_size) { 1648 write_fragment(*fragment); 1649 *fragment = NULL; 1650 } 1651 1652 ffrg = malloc(sizeof(struct fragment)); 1653 if(ffrg == NULL) 1654 MEM_ERROR(); 1655 1656 if(*fragment == NULL) 1657 *fragment = allocate_fragment(); 1658 1659 ffrg->index = (*fragment)->block; 1660 ffrg->offset = (*fragment)->size; 1661 ffrg->size = file_buffer->size; 1662 memcpy((*fragment)->data + (*fragment)->size, file_buffer->data, 1663 file_buffer->size); 1664 (*fragment)->size += file_buffer->size; 1665 1666 return ffrg; 1667 } 1668 1669 1670 long long generic_write_table(int length, void *buffer, int length2, 1671 void *buffer2, int uncompressed) 1672 { 1673 int meta_blocks = (length + SQUASHFS_METADATA_SIZE - 1) / 1674 SQUASHFS_METADATA_SIZE; 1675 long long *list, start_bytes; 1676 int compressed_size, i, list_size = meta_blocks * sizeof(long long); 1677 unsigned short c_byte; 1678 char cbuffer[(SQUASHFS_METADATA_SIZE << 2) + 2]; 1679 1680 #ifdef SQUASHFS_TRACE 1681 long long obytes = bytes; 1682 int olength = length; 1683 #endif 1684 1685 list = malloc(list_size); 1686 if(list == NULL) 1687 MEM_ERROR(); 1688 1689 for(i = 0; i < meta_blocks; i++) { 1690 int avail_bytes = length > SQUASHFS_METADATA_SIZE ? 1691 SQUASHFS_METADATA_SIZE : length; 1692 c_byte = mangle(cbuffer + BLOCK_OFFSET, buffer + i * 1693 SQUASHFS_METADATA_SIZE , avail_bytes, 1694 SQUASHFS_METADATA_SIZE, uncompressed, 0); 1695 SQUASHFS_SWAP_SHORTS(&c_byte, cbuffer, 1); 1696 list[i] = bytes; 1697 compressed_size = SQUASHFS_COMPRESSED_SIZE(c_byte) + 1698 BLOCK_OFFSET; 1699 TRACE("block %d @ 0x%llx, compressed size %d\n", i, bytes, 1700 compressed_size); 1701 write_destination(fd, bytes, compressed_size, cbuffer); 1702 bytes += compressed_size; 1703 total_bytes += avail_bytes; 1704 length -= avail_bytes; 1705 } 1706 1707 start_bytes = bytes; 1708 if(length2) { 1709 write_destination(fd, bytes, length2, buffer2); 1710 bytes += length2; 1711 total_bytes += length2; 1712 } 1713 1714 SQUASHFS_INSWAP_LONG_LONGS(list, meta_blocks); 1715 write_destination(fd, bytes, list_size, list); 1716 bytes += list_size; 1717 total_bytes += list_size; 1718 1719 TRACE("generic_write_table: total uncompressed %d compressed %lld\n", 1720 olength, bytes - obytes); 1721 1722 free(list); 1723 1724 return start_bytes; 1725 } 1726 1727 1728 long long write_fragment_table() 1729 { 1730 unsigned int frag_bytes = SQUASHFS_FRAGMENT_BYTES(fragments); 1731 int i; 1732 1733 TRACE("write_fragment_table: fragments %d, frag_bytes %d\n", fragments, 1734 frag_bytes); 1735 for(i = 0; i < fragments; i++) { 1736 TRACE("write_fragment_table: fragment %d, start_block 0x%llx, " 1737 "size %d\n", i, fragment_table[i].start_block, 1738 fragment_table[i].size); 1739 SQUASHFS_INSWAP_FRAGMENT_ENTRY(&fragment_table[i]); 1740 } 1741 1742 return generic_write_table(frag_bytes, fragment_table, 0, NULL, noF); 1743 } 1744 1745 1746 char read_from_file_buffer[SQUASHFS_FILE_MAX_SIZE]; 1747 static char *read_from_disk(long long start, unsigned int avail_bytes) 1748 { 1749 int res; 1750 1751 res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer); 1752 if(res == 0) 1753 return NULL; 1754 1755 return read_from_file_buffer; 1756 } 1757 1758 1759 char read_from_file_buffer2[SQUASHFS_FILE_MAX_SIZE]; 1760 char *read_from_disk2(long long start, unsigned int avail_bytes) 1761 { 1762 int res; 1763 1764 res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer2); 1765 if(res == 0) 1766 return NULL; 1767 1768 return read_from_file_buffer2; 1769 } 1770 1771 1772 /* 1773 * Compute 16 bit BSD checksum over the data 1774 */ 1775 unsigned short get_checksum(char *buff, int bytes, unsigned short chksum) 1776 { 1777 unsigned char *b = (unsigned char *) buff; 1778 1779 while(bytes --) { 1780 chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1; 1781 chksum += *b++; 1782 } 1783 1784 return chksum; 1785 } 1786 1787 1788 unsigned short get_checksum_disk(long long start, long long l, 1789 unsigned int *blocks) 1790 { 1791 unsigned short chksum = 0; 1792 unsigned int bytes; 1793 struct file_buffer *write_buffer; 1794 int i; 1795 1796 for(i = 0; l; i++) { 1797 bytes = SQUASHFS_COMPRESSED_SIZE_BLOCK(blocks[i]); 1798 if(bytes == 0) /* sparse block */ 1799 continue; 1800 write_buffer = cache_lookup(bwriter_buffer, start); 1801 if(write_buffer) { 1802 chksum = get_checksum(write_buffer->data, bytes, 1803 chksum); 1804 cache_block_put(write_buffer); 1805 } else { 1806 void *data = read_from_disk(start, bytes); 1807 if(data == NULL) { 1808 ERROR("Failed to checksum data from output" 1809 " filesystem\n"); 1810 BAD_ERROR("Output filesystem corrupted?\n"); 1811 } 1812 1813 chksum = get_checksum(data, bytes, chksum); 1814 } 1815 1816 l -= bytes; 1817 start += bytes; 1818 } 1819 1820 return chksum; 1821 } 1822 1823 1824 unsigned short get_checksum_mem(char *buff, int bytes) 1825 { 1826 return get_checksum(buff, bytes, 0); 1827 } 1828 1829 1830 unsigned short get_checksum_mem_buffer(struct file_buffer *file_buffer) 1831 { 1832 if(file_buffer == NULL) 1833 return 0; 1834 else 1835 return get_checksum(file_buffer->data, file_buffer->size, 0); 1836 } 1837 1838 1839 #define DUP_HASH(a) (a & 0xffff) 1840 void add_file(long long start, long long file_size, long long file_bytes, 1841 unsigned int *block_listp, int blocks, unsigned int fragment, 1842 int offset, int bytes) 1843 { 1844 struct fragment *frg; 1845 unsigned int *block_list = block_listp; 1846 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1847 struct append_file *append_file; 1848 struct file_info *file; 1849 1850 if(!duplicate_checking || file_size == 0) 1851 return; 1852 1853 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) { 1854 if(file_size != dupl_ptr->file_size) 1855 continue; 1856 if(blocks != 0 && start != dupl_ptr->start) 1857 continue; 1858 if(fragment != dupl_ptr->fragment->index) 1859 continue; 1860 if(fragment != SQUASHFS_INVALID_FRAG && (offset != 1861 dupl_ptr->fragment->offset || bytes != 1862 dupl_ptr->fragment->size)) 1863 continue; 1864 return; 1865 } 1866 1867 frg = malloc(sizeof(struct fragment)); 1868 if(frg == NULL) 1869 MEM_ERROR(); 1870 1871 frg->index = fragment; 1872 frg->offset = offset; 1873 frg->size = bytes; 1874 1875 file = add_non_dup(file_size, file_bytes, block_list, start, frg, 0, 0, 1876 FALSE, FALSE); 1877 1878 if(fragment == SQUASHFS_INVALID_FRAG) 1879 return; 1880 1881 append_file = malloc(sizeof(struct append_file)); 1882 if(append_file == NULL) 1883 MEM_ERROR(); 1884 1885 append_file->file = file; 1886 append_file->next = file_mapping[fragment]; 1887 file_mapping[fragment] = append_file; 1888 } 1889 1890 1891 int pre_duplicate(long long file_size) 1892 { 1893 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1894 1895 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) 1896 if(dupl_ptr->file_size == file_size) 1897 return TRUE; 1898 1899 return FALSE; 1900 } 1901 1902 1903 struct file_info *add_non_dup(long long file_size, long long bytes, 1904 unsigned int *block_list, long long start, struct fragment *fragment, 1905 unsigned short checksum, unsigned short fragment_checksum, 1906 int checksum_flag, int checksum_frag_flag) 1907 { 1908 struct file_info *dupl_ptr = malloc(sizeof(struct file_info)); 1909 1910 if(dupl_ptr == NULL) 1911 MEM_ERROR(); 1912 1913 dupl_ptr->file_size = file_size; 1914 dupl_ptr->bytes = bytes; 1915 dupl_ptr->block_list = block_list; 1916 dupl_ptr->start = start; 1917 dupl_ptr->fragment = fragment; 1918 dupl_ptr->checksum = checksum; 1919 dupl_ptr->fragment_checksum = fragment_checksum; 1920 dupl_ptr->have_frag_checksum = checksum_frag_flag; 1921 dupl_ptr->have_checksum = checksum_flag; 1922 1923 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1924 pthread_mutex_lock(&dup_mutex); 1925 dupl_ptr->next = dupl[DUP_HASH(file_size)]; 1926 dupl[DUP_HASH(file_size)] = dupl_ptr; 1927 dup_files ++; 1928 pthread_cleanup_pop(1); 1929 1930 return dupl_ptr; 1931 } 1932 1933 1934 struct fragment *frag_duplicate(struct file_buffer *file_buffer, char *dont_put) 1935 { 1936 struct file_info *dupl_ptr; 1937 struct file_buffer *buffer; 1938 struct file_info *dupl_start = file_buffer->dupl_start; 1939 long long file_size = file_buffer->file_size; 1940 unsigned short checksum = file_buffer->checksum; 1941 int res; 1942 1943 if(file_buffer->duplicate) { 1944 TRACE("Found duplicate file, fragment %d, size %d, offset %d, " 1945 "checksum 0x%x\n", dupl_start->fragment->index, 1946 file_size, dupl_start->fragment->offset, checksum); 1947 *dont_put = TRUE; 1948 return dupl_start->fragment; 1949 } else { 1950 *dont_put = FALSE; 1951 dupl_ptr = dupl[DUP_HASH(file_size)]; 1952 } 1953 1954 for(; dupl_ptr && dupl_ptr != dupl_start; dupl_ptr = dupl_ptr->next) { 1955 if(file_size == dupl_ptr->file_size && file_size == 1956 dupl_ptr->fragment->size) { 1957 if(get_fragment_checksum(dupl_ptr) == checksum) { 1958 buffer = get_fragment(dupl_ptr->fragment); 1959 res = memcmp(file_buffer->data, buffer->data + 1960 dupl_ptr->fragment->offset, file_size); 1961 cache_block_put(buffer); 1962 if(res == 0) 1963 break; 1964 } 1965 } 1966 } 1967 1968 if(!dupl_ptr || dupl_ptr == dupl_start) 1969 return NULL; 1970 1971 TRACE("Found duplicate file, fragment %d, size %d, offset %d, " 1972 "checksum 0x%x\n", dupl_ptr->fragment->index, file_size, 1973 dupl_ptr->fragment->offset, checksum); 1974 1975 return dupl_ptr->fragment; 1976 } 1977 1978 1979 struct file_info *duplicate(long long file_size, long long bytes, 1980 unsigned int **block_list, long long *start, struct fragment **fragment, 1981 struct file_buffer *file_buffer, int blocks, unsigned short checksum, 1982 int checksum_flag) 1983 { 1984 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1985 int frag_bytes = file_buffer ? file_buffer->size : 0; 1986 unsigned short fragment_checksum = file_buffer ? 1987 file_buffer->checksum : 0; 1988 1989 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) 1990 if(file_size == dupl_ptr->file_size && bytes == dupl_ptr->bytes 1991 && frag_bytes == dupl_ptr->fragment->size) { 1992 long long target_start, dup_start = dupl_ptr->start; 1993 int block; 1994 1995 if(memcmp(*block_list, dupl_ptr->block_list, blocks * 1996 sizeof(unsigned int)) != 0) 1997 continue; 1998 1999 if(checksum_flag == FALSE) { 2000 checksum = get_checksum_disk(*start, bytes, 2001 *block_list); 2002 checksum_flag = TRUE; 2003 } 2004 2005 if(!dupl_ptr->have_checksum) { 2006 dupl_ptr->checksum = 2007 get_checksum_disk(dupl_ptr->start, 2008 dupl_ptr->bytes, dupl_ptr->block_list); 2009 dupl_ptr->have_checksum = TRUE; 2010 } 2011 2012 if(checksum != dupl_ptr->checksum || 2013 fragment_checksum != 2014 get_fragment_checksum(dupl_ptr)) 2015 continue; 2016 2017 target_start = *start; 2018 for(block = 0; block < blocks; block ++) { 2019 int size = SQUASHFS_COMPRESSED_SIZE_BLOCK 2020 ((*block_list)[block]); 2021 struct file_buffer *target_buffer = NULL; 2022 struct file_buffer *dup_buffer = NULL; 2023 char *target_data, *dup_data; 2024 int res; 2025 2026 if(size == 0) 2027 continue; 2028 target_buffer = cache_lookup(bwriter_buffer, 2029 target_start); 2030 if(target_buffer) 2031 target_data = target_buffer->data; 2032 else { 2033 target_data = 2034 read_from_disk(target_start, 2035 size); 2036 if(target_data == NULL) { 2037 ERROR("Failed to read data from" 2038 " output filesystem\n"); 2039 BAD_ERROR("Output filesystem" 2040 " corrupted?\n"); 2041 } 2042 } 2043 2044 dup_buffer = cache_lookup(bwriter_buffer, 2045 dup_start); 2046 if(dup_buffer) 2047 dup_data = dup_buffer->data; 2048 else { 2049 dup_data = read_from_disk2(dup_start, 2050 size); 2051 if(dup_data == NULL) { 2052 ERROR("Failed to read data from" 2053 " output filesystem\n"); 2054 BAD_ERROR("Output filesystem" 2055 " corrupted?\n"); 2056 } 2057 } 2058 2059 res = memcmp(target_data, dup_data, size); 2060 cache_block_put(target_buffer); 2061 cache_block_put(dup_buffer); 2062 if(res != 0) 2063 break; 2064 target_start += size; 2065 dup_start += size; 2066 } 2067 if(block == blocks) { 2068 struct file_buffer *frag_buffer = 2069 get_fragment(dupl_ptr->fragment); 2070 2071 if(frag_bytes == 0 || 2072 memcmp(file_buffer->data, 2073 frag_buffer->data + 2074 dupl_ptr->fragment->offset, 2075 frag_bytes) == 0) { 2076 TRACE("Found duplicate file, start " 2077 "0x%llx, size %lld, checksum " 2078 "0x%x, fragment %d, size %d, " 2079 "offset %d, checksum 0x%x\n", 2080 dupl_ptr->start, 2081 dupl_ptr->bytes, 2082 dupl_ptr->checksum, 2083 dupl_ptr->fragment->index, 2084 frag_bytes, 2085 dupl_ptr->fragment->offset, 2086 fragment_checksum); 2087 *block_list = dupl_ptr->block_list; 2088 *start = dupl_ptr->start; 2089 *fragment = dupl_ptr->fragment; 2090 cache_block_put(frag_buffer); 2091 return 0; 2092 } 2093 cache_block_put(frag_buffer); 2094 } 2095 } 2096 2097 2098 return add_non_dup(file_size, bytes, *block_list, *start, *fragment, 2099 checksum, fragment_checksum, checksum_flag, TRUE); 2100 } 2101 2102 2103 static inline int is_fragment(struct inode_info *inode) 2104 { 2105 off_t file_size = inode->buf.st_size; 2106 2107 /* 2108 * If this block is to be compressed differently to the 2109 * fragment compression then it cannot be a fragment 2110 */ 2111 if(inode->noF != noF) 2112 return FALSE; 2113 2114 return !inode->no_fragments && file_size && (file_size < block_size || 2115 (inode->always_use_fragments && file_size & (block_size - 1))); 2116 } 2117 2118 2119 void put_file_buffer(struct file_buffer *file_buffer) 2120 { 2121 /* 2122 * Decide where to send the file buffer: 2123 * - compressible non-fragment blocks go to the deflate threads, 2124 * - fragments go to the process fragment threads, 2125 * - all others go directly to the main thread 2126 */ 2127 if(file_buffer->error) { 2128 file_buffer->fragment = 0; 2129 seq_queue_put(to_main, file_buffer); 2130 } else if (file_buffer->file_size == 0) 2131 seq_queue_put(to_main, file_buffer); 2132 else if(file_buffer->fragment) 2133 queue_put(to_process_frag, file_buffer); 2134 else 2135 queue_put(to_deflate, file_buffer); 2136 } 2137 2138 2139 static int seq = 0; 2140 void reader_read_process(struct dir_ent *dir_ent) 2141 { 2142 long long bytes = 0; 2143 struct inode_info *inode = dir_ent->inode; 2144 struct file_buffer *prev_buffer = NULL, *file_buffer; 2145 int status, byte, res, child; 2146 int file = pseudo_exec_file(get_pseudo_file(inode->pseudo_id), &child); 2147 2148 if(!file) { 2149 file_buffer = cache_get_nohash(reader_buffer); 2150 file_buffer->sequence = seq ++; 2151 goto read_err; 2152 } 2153 2154 while(1) { 2155 file_buffer = cache_get_nohash(reader_buffer); 2156 file_buffer->sequence = seq ++; 2157 file_buffer->noD = inode->noD; 2158 2159 byte = read_bytes(file, file_buffer->data, block_size); 2160 if(byte == -1) 2161 goto read_err2; 2162 2163 file_buffer->size = byte; 2164 file_buffer->file_size = -1; 2165 file_buffer->error = FALSE; 2166 file_buffer->fragment = FALSE; 2167 bytes += byte; 2168 2169 if(byte == 0) 2170 break; 2171 2172 /* 2173 * Update progress bar size. This is done 2174 * on every block rather than waiting for all blocks to be 2175 * read incase write_file_process() is running in parallel 2176 * with this. Otherwise the current progress bar position 2177 * may get ahead of the progress bar size. 2178 */ 2179 progress_bar_size(1); 2180 2181 if(prev_buffer) 2182 put_file_buffer(prev_buffer); 2183 prev_buffer = file_buffer; 2184 } 2185 2186 /* 2187 * Update inode file size now that the size of the dynamic pseudo file 2188 * is known. This is needed for the -info option. 2189 */ 2190 inode->buf.st_size = bytes; 2191 2192 res = waitpid(child, &status, 0); 2193 close(file); 2194 2195 if(res == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) 2196 goto read_err; 2197 2198 if(prev_buffer == NULL) 2199 prev_buffer = file_buffer; 2200 else { 2201 cache_block_put(file_buffer); 2202 seq --; 2203 } 2204 prev_buffer->file_size = bytes; 2205 prev_buffer->fragment = is_fragment(inode); 2206 put_file_buffer(prev_buffer); 2207 2208 return; 2209 2210 read_err2: 2211 close(file); 2212 read_err: 2213 if(prev_buffer) { 2214 cache_block_put(file_buffer); 2215 seq --; 2216 file_buffer = prev_buffer; 2217 } 2218 file_buffer->error = TRUE; 2219 put_file_buffer(file_buffer); 2220 } 2221 2222 2223 void reader_read_file(struct dir_ent *dir_ent) 2224 { 2225 struct stat *buf = &dir_ent->inode->buf, buf2; 2226 struct file_buffer *file_buffer; 2227 int blocks, file, res; 2228 long long bytes, read_size; 2229 struct inode_info *inode = dir_ent->inode; 2230 2231 if(inode->read) 2232 return; 2233 2234 inode->read = TRUE; 2235 again: 2236 bytes = 0; 2237 read_size = buf->st_size; 2238 blocks = (read_size + block_size - 1) >> block_log; 2239 2240 file = open(pathname_reader(dir_ent), O_RDONLY); 2241 if(file == -1) { 2242 file_buffer = cache_get_nohash(reader_buffer); 2243 file_buffer->sequence = seq ++; 2244 goto read_err2; 2245 } 2246 2247 do { 2248 file_buffer = cache_get_nohash(reader_buffer); 2249 file_buffer->file_size = read_size; 2250 file_buffer->sequence = seq ++; 2251 file_buffer->noD = inode->noD; 2252 file_buffer->error = FALSE; 2253 2254 /* 2255 * Always try to read block_size bytes from the file rather 2256 * than expected bytes (which will be less than the block_size 2257 * at the file tail) to check that the file hasn't grown 2258 * since being stated. If it is longer (or shorter) than 2259 * expected, then restat, and try again. Note the special 2260 * case where the file is an exact multiple of the block_size 2261 * is dealt with later. 2262 */ 2263 file_buffer->size = read_bytes(file, file_buffer->data, 2264 block_size); 2265 if(file_buffer->size == -1) 2266 goto read_err; 2267 2268 bytes += file_buffer->size; 2269 2270 if(blocks > 1) { 2271 /* non-tail block should be exactly block_size */ 2272 if(file_buffer->size < block_size) 2273 goto restat; 2274 2275 file_buffer->fragment = FALSE; 2276 put_file_buffer(file_buffer); 2277 } 2278 } while(-- blocks > 0); 2279 2280 /* Overall size including tail should match */ 2281 if(read_size != bytes) 2282 goto restat; 2283 2284 if(read_size && read_size % block_size == 0) { 2285 /* 2286 * Special case where we've not tried to read past the end of 2287 * the file. We expect to get EOF, i.e. the file isn't larger 2288 * than we expect. 2289 */ 2290 char buffer; 2291 int res; 2292 2293 res = read_bytes(file, &buffer, 1); 2294 if(res == -1) 2295 goto read_err; 2296 2297 if(res != 0) 2298 goto restat; 2299 } 2300 2301 file_buffer->fragment = is_fragment(inode); 2302 put_file_buffer(file_buffer); 2303 2304 close(file); 2305 2306 return; 2307 2308 restat: 2309 res = fstat(file, &buf2); 2310 if(res == -1) { 2311 ERROR("Cannot stat dir/file %s because %s\n", 2312 pathname_reader(dir_ent), strerror(errno)); 2313 goto read_err; 2314 } 2315 2316 if(read_size != buf2.st_size) { 2317 close(file); 2318 memcpy(buf, &buf2, sizeof(struct stat)); 2319 file_buffer->error = 2; 2320 put_file_buffer(file_buffer); 2321 goto again; 2322 } 2323 read_err: 2324 close(file); 2325 read_err2: 2326 file_buffer->error = TRUE; 2327 put_file_buffer(file_buffer); 2328 } 2329 2330 2331 void reader_scan(struct dir_info *dir) { 2332 struct dir_ent *dir_ent = dir->list; 2333 2334 for(; dir_ent; dir_ent = dir_ent->next) { 2335 struct stat *buf = &dir_ent->inode->buf; 2336 if(dir_ent->inode->root_entry) 2337 continue; 2338 2339 if(IS_PSEUDO_PROCESS(dir_ent->inode)) { 2340 reader_read_process(dir_ent); 2341 continue; 2342 } 2343 2344 switch(buf->st_mode & S_IFMT) { 2345 case S_IFREG: 2346 reader_read_file(dir_ent); 2347 break; 2348 case S_IFDIR: 2349 reader_scan(dir_ent->dir); 2350 break; 2351 } 2352 } 2353 } 2354 2355 2356 void *reader(void *arg) 2357 { 2358 if(!sorted) 2359 reader_scan(queue_get(to_reader)); 2360 else { 2361 int i; 2362 struct priority_entry *entry; 2363 2364 queue_get(to_reader); 2365 for(i = 65535; i >= 0; i--) 2366 for(entry = priority_list[i]; entry; 2367 entry = entry->next) 2368 reader_read_file(entry->dir); 2369 } 2370 2371 pthread_exit(NULL); 2372 } 2373 2374 2375 void *writer(void *arg) 2376 { 2377 while(1) { 2378 struct file_buffer *file_buffer = queue_get(to_writer); 2379 off_t off; 2380 2381 if(file_buffer == NULL) { 2382 queue_put(from_writer, NULL); 2383 continue; 2384 } 2385 2386 off = file_buffer->block; 2387 2388 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 2389 pthread_mutex_lock(&pos_mutex); 2390 2391 if(lseek(fd, off, SEEK_SET) == -1) { 2392 ERROR("writer: Lseek on destination failed because " 2393 "%s, offset=0x%llx\n", strerror(errno), off); 2394 BAD_ERROR("Probably out of space on output " 2395 "%s\n", block_device ? "block device" : 2396 "filesystem"); 2397 } 2398 2399 if(write_bytes(fd, file_buffer->data, 2400 file_buffer->size) == -1) 2401 BAD_ERROR("Failed to write to output %s\n", 2402 block_device ? "block device" : "filesystem"); 2403 2404 pthread_cleanup_pop(1); 2405 2406 cache_block_put(file_buffer); 2407 } 2408 } 2409 2410 2411 int all_zero(struct file_buffer *file_buffer) 2412 { 2413 int i; 2414 long entries = file_buffer->size / sizeof(long); 2415 long *p = (long *) file_buffer->data; 2416 2417 for(i = 0; i < entries && p[i] == 0; i++); 2418 2419 if(i == entries) { 2420 for(i = file_buffer->size & ~(sizeof(long) - 1); 2421 i < file_buffer->size && file_buffer->data[i] == 0; 2422 i++); 2423 2424 return i == file_buffer->size; 2425 } 2426 2427 return 0; 2428 } 2429 2430 2431 void *deflator(void *arg) 2432 { 2433 struct file_buffer *write_buffer = cache_get_nohash(bwriter_buffer); 2434 void *stream = NULL; 2435 int res; 2436 2437 res = compressor_init(comp, &stream, block_size, 1); 2438 if(res) 2439 BAD_ERROR("deflator:: compressor_init failed\n"); 2440 2441 while(1) { 2442 struct file_buffer *file_buffer = queue_get(to_deflate); 2443 2444 if(sparse_files && all_zero(file_buffer)) { 2445 file_buffer->c_byte = 0; 2446 seq_queue_put(to_main, file_buffer); 2447 } else { 2448 write_buffer->c_byte = mangle2(stream, 2449 write_buffer->data, file_buffer->data, 2450 file_buffer->size, block_size, 2451 file_buffer->noD, 1); 2452 write_buffer->sequence = file_buffer->sequence; 2453 write_buffer->file_size = file_buffer->file_size; 2454 write_buffer->block = file_buffer->block; 2455 write_buffer->size = SQUASHFS_COMPRESSED_SIZE_BLOCK 2456 (write_buffer->c_byte); 2457 write_buffer->fragment = FALSE; 2458 write_buffer->error = FALSE; 2459 cache_block_put(file_buffer); 2460 seq_queue_put(to_main, write_buffer); 2461 write_buffer = cache_get_nohash(bwriter_buffer); 2462 } 2463 } 2464 } 2465 2466 2467 void *frag_deflator(void *arg) 2468 { 2469 void *stream = NULL; 2470 int res; 2471 2472 res = compressor_init(comp, &stream, block_size, 1); 2473 if(res) 2474 BAD_ERROR("frag_deflator:: compressor_init failed\n"); 2475 2476 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 2477 2478 while(1) { 2479 int c_byte, compressed_size; 2480 struct file_buffer *file_buffer = queue_get(to_frag); 2481 struct file_buffer *write_buffer = 2482 cache_get(fwriter_buffer, file_buffer->block); 2483 2484 c_byte = mangle2(stream, write_buffer->data, file_buffer->data, 2485 file_buffer->size, block_size, noF, 1); 2486 compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); 2487 write_buffer->size = compressed_size; 2488 pthread_mutex_lock(&fragment_mutex); 2489 if(fragments_locked == FALSE) { 2490 fragment_table[file_buffer->block].size = c_byte; 2491 fragment_table[file_buffer->block].start_block = bytes; 2492 write_buffer->block = bytes; 2493 bytes += compressed_size; 2494 fragments_outstanding --; 2495 queue_put(to_writer, write_buffer); 2496 pthread_mutex_unlock(&fragment_mutex); 2497 TRACE("Writing fragment %lld, uncompressed size %d, " 2498 "compressed size %d\n", file_buffer->block, 2499 file_buffer->size, compressed_size); 2500 } else { 2501 add_pending_fragment(write_buffer, c_byte, 2502 file_buffer->block); 2503 pthread_mutex_unlock(&fragment_mutex); 2504 } 2505 cache_block_put(file_buffer); 2506 } 2507 2508 pthread_cleanup_pop(0); 2509 } 2510 2511 2512 struct file_buffer *get_file_buffer() 2513 { 2514 struct file_buffer *file_buffer = seq_queue_get(to_main); 2515 2516 return file_buffer; 2517 } 2518 2519 2520 void write_file_empty(squashfs_inode *inode, struct dir_ent *dir_ent, 2521 struct file_buffer *file_buffer, int *duplicate_file) 2522 { 2523 file_count ++; 2524 *duplicate_file = FALSE; 2525 cache_block_put(file_buffer); 2526 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, 0, 0, 0, 2527 NULL, &empty_fragment, NULL, 0); 2528 } 2529 2530 2531 void write_file_frag(squashfs_inode *inode, struct dir_ent *dir_ent, 2532 struct file_buffer *file_buffer, int *duplicate_file) 2533 { 2534 int size = file_buffer->file_size; 2535 struct fragment *fragment; 2536 unsigned short checksum = file_buffer->checksum; 2537 char dont_put; 2538 2539 fragment = frag_duplicate(file_buffer, &dont_put); 2540 *duplicate_file = !fragment; 2541 if(!fragment) { 2542 fragment = get_and_fill_fragment(file_buffer, dir_ent); 2543 if(duplicate_checking) 2544 add_non_dup(size, 0, NULL, 0, fragment, 0, checksum, 2545 TRUE, TRUE); 2546 } 2547 2548 if(dont_put) 2549 free(file_buffer); 2550 else 2551 cache_block_put(file_buffer); 2552 2553 total_bytes += size; 2554 file_count ++; 2555 2556 inc_progress_bar(); 2557 2558 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, size, 0, 2559 0, NULL, fragment, NULL, 0); 2560 2561 if(!duplicate_checking) 2562 free_fragment(fragment); 2563 } 2564 2565 2566 int write_file_process(squashfs_inode *inode, struct dir_ent *dir_ent, 2567 struct file_buffer *read_buffer, int *duplicate_file) 2568 { 2569 long long read_size, file_bytes, start; 2570 struct fragment *fragment; 2571 unsigned int *block_list = NULL; 2572 int block = 0, status; 2573 long long sparse = 0; 2574 struct file_buffer *fragment_buffer = NULL; 2575 2576 *duplicate_file = FALSE; 2577 2578 lock_fragments(); 2579 2580 file_bytes = 0; 2581 start = bytes; 2582 while (1) { 2583 read_size = read_buffer->file_size; 2584 if(read_buffer->fragment) 2585 fragment_buffer = read_buffer; 2586 else { 2587 block_list = realloc(block_list, (block + 1) * 2588 sizeof(unsigned int)); 2589 if(block_list == NULL) 2590 MEM_ERROR(); 2591 block_list[block ++] = read_buffer->c_byte; 2592 if(read_buffer->c_byte) { 2593 read_buffer->block = bytes; 2594 bytes += read_buffer->size; 2595 cache_hash(read_buffer, read_buffer->block); 2596 file_bytes += read_buffer->size; 2597 queue_put(to_writer, read_buffer); 2598 } else { 2599 sparse += read_buffer->size; 2600 cache_block_put(read_buffer); 2601 } 2602 } 2603 inc_progress_bar(); 2604 2605 if(read_size != -1) 2606 break; 2607 2608 read_buffer = get_file_buffer(); 2609 if(read_buffer->error) 2610 goto read_err; 2611 } 2612 2613 unlock_fragments(); 2614 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2615 2616 if(duplicate_checking) 2617 add_non_dup(read_size, file_bytes, block_list, start, fragment, 2618 0, fragment_buffer ? fragment_buffer->checksum : 0, 2619 FALSE, TRUE); 2620 cache_block_put(fragment_buffer); 2621 file_count ++; 2622 total_bytes += read_size; 2623 2624 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, 2625 block, block_list, fragment, NULL, sparse); 2626 2627 if(duplicate_checking == FALSE) { 2628 free(block_list); 2629 free_fragment(fragment); 2630 } 2631 2632 return 0; 2633 2634 read_err: 2635 dec_progress_bar(block); 2636 status = read_buffer->error; 2637 bytes = start; 2638 if(!block_device) { 2639 int res; 2640 2641 queue_put(to_writer, NULL); 2642 if(queue_get(from_writer) != 0) 2643 EXIT_MKSQUASHFS(); 2644 res = ftruncate(fd, bytes); 2645 if(res != 0) 2646 BAD_ERROR("Failed to truncate dest file because %s\n", 2647 strerror(errno)); 2648 } 2649 unlock_fragments(); 2650 free(block_list); 2651 cache_block_put(read_buffer); 2652 return status; 2653 } 2654 2655 2656 int write_file_blocks_dup(squashfs_inode *inode, struct dir_ent *dir_ent, 2657 struct file_buffer *read_buffer, int *duplicate_file) 2658 { 2659 int block, thresh; 2660 long long read_size = read_buffer->file_size; 2661 long long file_bytes, dup_start, start; 2662 struct fragment *fragment; 2663 struct file_info *dupl_ptr; 2664 int blocks = (read_size + block_size - 1) >> block_log; 2665 unsigned int *block_list, *block_listp; 2666 struct file_buffer **buffer_list; 2667 int status; 2668 long long sparse = 0; 2669 struct file_buffer *fragment_buffer = NULL; 2670 2671 block_list = malloc(blocks * sizeof(unsigned int)); 2672 if(block_list == NULL) 2673 MEM_ERROR(); 2674 block_listp = block_list; 2675 2676 buffer_list = malloc(blocks * sizeof(struct file_buffer *)); 2677 if(buffer_list == NULL) 2678 MEM_ERROR(); 2679 2680 lock_fragments(); 2681 2682 file_bytes = 0; 2683 start = dup_start = bytes; 2684 thresh = blocks > bwriter_size ? blocks - bwriter_size : 0; 2685 2686 for(block = 0; block < blocks;) { 2687 if(read_buffer->fragment) { 2688 block_list[block] = 0; 2689 buffer_list[block] = NULL; 2690 fragment_buffer = read_buffer; 2691 blocks = read_size >> block_log; 2692 } else { 2693 block_list[block] = read_buffer->c_byte; 2694 2695 if(read_buffer->c_byte) { 2696 read_buffer->block = bytes; 2697 bytes += read_buffer->size; 2698 file_bytes += read_buffer->size; 2699 cache_hash(read_buffer, read_buffer->block); 2700 if(block < thresh) { 2701 buffer_list[block] = NULL; 2702 queue_put(to_writer, read_buffer); 2703 } else 2704 buffer_list[block] = read_buffer; 2705 } else { 2706 buffer_list[block] = NULL; 2707 sparse += read_buffer->size; 2708 cache_block_put(read_buffer); 2709 } 2710 } 2711 inc_progress_bar(); 2712 2713 if(++block < blocks) { 2714 read_buffer = get_file_buffer(); 2715 if(read_buffer->error) 2716 goto read_err; 2717 } 2718 } 2719 2720 dupl_ptr = duplicate(read_size, file_bytes, &block_listp, &dup_start, 2721 &fragment, fragment_buffer, blocks, 0, FALSE); 2722 2723 if(dupl_ptr) { 2724 *duplicate_file = FALSE; 2725 for(block = thresh; block < blocks; block ++) 2726 if(buffer_list[block]) 2727 queue_put(to_writer, buffer_list[block]); 2728 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2729 dupl_ptr->fragment = fragment; 2730 } else { 2731 *duplicate_file = TRUE; 2732 for(block = thresh; block < blocks; block ++) 2733 cache_block_put(buffer_list[block]); 2734 bytes = start; 2735 if(thresh && !block_device) { 2736 int res; 2737 2738 queue_put(to_writer, NULL); 2739 if(queue_get(from_writer) != 0) 2740 EXIT_MKSQUASHFS(); 2741 res = ftruncate(fd, bytes); 2742 if(res != 0) 2743 BAD_ERROR("Failed to truncate dest file because" 2744 " %s\n", strerror(errno)); 2745 } 2746 } 2747 2748 unlock_fragments(); 2749 cache_block_put(fragment_buffer); 2750 free(buffer_list); 2751 file_count ++; 2752 total_bytes += read_size; 2753 2754 /* 2755 * sparse count is needed to ensure squashfs correctly reports a 2756 * a smaller block count on stat calls to sparse files. This is 2757 * to ensure intelligent applications like cp correctly handle the 2758 * file as a sparse file. If the file in the original filesystem isn't 2759 * stored as a sparse file then still store it sparsely in squashfs, but 2760 * report it as non-sparse on stat calls to preserve semantics 2761 */ 2762 if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) 2763 sparse = 0; 2764 2765 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, 2766 dup_start, blocks, block_listp, fragment, NULL, sparse); 2767 2768 if(*duplicate_file == TRUE) 2769 free(block_list); 2770 2771 return 0; 2772 2773 read_err: 2774 dec_progress_bar(block); 2775 status = read_buffer->error; 2776 bytes = start; 2777 if(thresh && !block_device) { 2778 int res; 2779 2780 queue_put(to_writer, NULL); 2781 if(queue_get(from_writer) != 0) 2782 EXIT_MKSQUASHFS(); 2783 res = ftruncate(fd, bytes); 2784 if(res != 0) 2785 BAD_ERROR("Failed to truncate dest file because %s\n", 2786 strerror(errno)); 2787 } 2788 unlock_fragments(); 2789 for(blocks = thresh; blocks < block; blocks ++) 2790 cache_block_put(buffer_list[blocks]); 2791 free(buffer_list); 2792 free(block_list); 2793 cache_block_put(read_buffer); 2794 return status; 2795 } 2796 2797 2798 int write_file_blocks(squashfs_inode *inode, struct dir_ent *dir_ent, 2799 struct file_buffer *read_buffer, int *dup) 2800 { 2801 long long read_size = read_buffer->file_size; 2802 long long file_bytes, start; 2803 struct fragment *fragment; 2804 unsigned int *block_list; 2805 int block, status; 2806 int blocks = (read_size + block_size - 1) >> block_log; 2807 long long sparse = 0; 2808 struct file_buffer *fragment_buffer = NULL; 2809 2810 if(pre_duplicate(read_size)) 2811 return write_file_blocks_dup(inode, dir_ent, read_buffer, dup); 2812 2813 *dup = FALSE; 2814 2815 block_list = malloc(blocks * sizeof(unsigned int)); 2816 if(block_list == NULL) 2817 MEM_ERROR(); 2818 2819 lock_fragments(); 2820 2821 file_bytes = 0; 2822 /* ANDROID CHANGES START*/ 2823 #ifdef ANDROID 2824 if (align_4k_blocks && bytes % 4096) { 2825 bytes += 4096 - (bytes % 4096); 2826 } 2827 #endif 2828 /* ANDROID CHANGES END */ 2829 start = bytes; 2830 for(block = 0; block < blocks;) { 2831 if(read_buffer->fragment) { 2832 block_list[block] = 0; 2833 fragment_buffer = read_buffer; 2834 blocks = read_size >> block_log; 2835 } else { 2836 block_list[block] = read_buffer->c_byte; 2837 if(read_buffer->c_byte) { 2838 read_buffer->block = bytes; 2839 bytes += read_buffer->size; 2840 cache_hash(read_buffer, read_buffer->block); 2841 file_bytes += read_buffer->size; 2842 queue_put(to_writer, read_buffer); 2843 } else { 2844 sparse += read_buffer->size; 2845 cache_block_put(read_buffer); 2846 } 2847 } 2848 inc_progress_bar(); 2849 2850 if(++block < blocks) { 2851 read_buffer = get_file_buffer(); 2852 if(read_buffer->error) 2853 goto read_err; 2854 } 2855 } 2856 2857 unlock_fragments(); 2858 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2859 2860 if(duplicate_checking) 2861 add_non_dup(read_size, file_bytes, block_list, start, fragment, 2862 0, fragment_buffer ? fragment_buffer->checksum : 0, 2863 FALSE, TRUE); 2864 cache_block_put(fragment_buffer); 2865 file_count ++; 2866 total_bytes += read_size; 2867 2868 /* 2869 * sparse count is needed to ensure squashfs correctly reports a 2870 * a smaller block count on stat calls to sparse files. This is 2871 * to ensure intelligent applications like cp correctly handle the 2872 * file as a sparse file. If the file in the original filesystem isn't 2873 * stored as a sparse file then still store it sparsely in squashfs, but 2874 * report it as non-sparse on stat calls to preserve semantics 2875 */ 2876 if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) 2877 sparse = 0; 2878 2879 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, 2880 blocks, block_list, fragment, NULL, sparse); 2881 2882 if(duplicate_checking == FALSE) { 2883 free(block_list); 2884 free_fragment(fragment); 2885 } 2886 2887 return 0; 2888 2889 read_err: 2890 dec_progress_bar(block); 2891 status = read_buffer->error; 2892 bytes = start; 2893 if(!block_device) { 2894 int res; 2895 2896 queue_put(to_writer, NULL); 2897 if(queue_get(from_writer) != 0) 2898 EXIT_MKSQUASHFS(); 2899 res = ftruncate(fd, bytes); 2900 if(res != 0) 2901 BAD_ERROR("Failed to truncate dest file because %s\n", 2902 strerror(errno)); 2903 } 2904 unlock_fragments(); 2905 free(block_list); 2906 cache_block_put(read_buffer); 2907 return status; 2908 } 2909 2910 2911 void write_file(squashfs_inode *inode, struct dir_ent *dir, int *dup) 2912 { 2913 int status; 2914 struct file_buffer *read_buffer; 2915 2916 again: 2917 read_buffer = get_file_buffer(); 2918 status = read_buffer->error; 2919 2920 if(status) 2921 cache_block_put(read_buffer); 2922 else if(read_buffer->file_size == -1) 2923 status = write_file_process(inode, dir, read_buffer, dup); 2924 else if(read_buffer->file_size == 0) 2925 write_file_empty(inode, dir, read_buffer, dup); 2926 else if(read_buffer->fragment && read_buffer->c_byte) 2927 write_file_frag(inode, dir, read_buffer, dup); 2928 else 2929 status = write_file_blocks(inode, dir, read_buffer, dup); 2930 2931 if(status == 2) { 2932 ERROR("File %s changed size while reading filesystem, " 2933 "attempting to re-read\n", pathname(dir)); 2934 goto again; 2935 } else if(status == 1) { 2936 ERROR_START("Failed to read file %s", pathname(dir)); 2937 ERROR_EXIT(", creating empty file\n"); 2938 write_file_empty(inode, dir, NULL, dup); 2939 } 2940 } 2941 2942 2943 #define BUFF_SIZE 512 2944 char *name; 2945 char *basename_r(); 2946 2947 char *getbase(char *pathname) 2948 { 2949 static char *b_buffer = NULL; 2950 static int b_size = BUFF_SIZE; 2951 char *result; 2952 2953 if(b_buffer == NULL) { 2954 b_buffer = malloc(b_size); 2955 if(b_buffer == NULL) 2956 MEM_ERROR(); 2957 } 2958 2959 while(1) { 2960 if(*pathname != '/') { 2961 result = getcwd(b_buffer, b_size); 2962 if(result == NULL && errno != ERANGE) 2963 BAD_ERROR("Getcwd failed in getbase\n"); 2964 2965 /* enough room for pathname + "/" + '\0' terminator? */ 2966 if(result && strlen(pathname) + 2 <= 2967 b_size - strlen(b_buffer)) { 2968 strcat(strcat(b_buffer, "/"), pathname); 2969 break; 2970 } 2971 } else if(strlen(pathname) < b_size) { 2972 strcpy(b_buffer, pathname); 2973 break; 2974 } 2975 2976 /* Buffer not large enough, realloc and try again */ 2977 b_buffer = realloc(b_buffer, b_size += BUFF_SIZE); 2978 if(b_buffer == NULL) 2979 MEM_ERROR(); 2980 } 2981 2982 name = b_buffer; 2983 if(((result = basename_r()) == NULL) || (strcmp(result, "..") == 0)) 2984 return NULL; 2985 else 2986 return result; 2987 } 2988 2989 2990 char *basename_r() 2991 { 2992 char *s; 2993 char *p; 2994 int n = 1; 2995 2996 for(;;) { 2997 s = name; 2998 if(*name == '\0') 2999 return NULL; 3000 if(*name != '/') { 3001 while(*name != '\0' && *name != '/') name++; 3002 n = name - s; 3003 } 3004 while(*name == '/') name++; 3005 if(strncmp(s, ".", n) == 0) 3006 continue; 3007 if((*name == '\0') || (strncmp(s, "..", n) == 0) || 3008 ((p = basename_r()) == NULL)) { 3009 s[n] = '\0'; 3010 return s; 3011 } 3012 if(strcmp(p, "..") == 0) 3013 continue; 3014 return p; 3015 } 3016 } 3017 3018 3019 struct inode_info *lookup_inode3(struct stat *buf, int pseudo, int id, 3020 char *symlink, int bytes) 3021 { 3022 int ino_hash = INODE_HASH(buf->st_dev, buf->st_ino); 3023 struct inode_info *inode; 3024 3025 /* 3026 * Look-up inode in hash table, if it already exists we have a 3027 * hard-link, so increment the nlink count and return it. 3028 * Don't do the look-up for directories because we don't hard-link 3029 * directories. 3030 */ 3031 if ((buf->st_mode & S_IFMT) != S_IFDIR) { 3032 for(inode = inode_info[ino_hash]; inode; inode = inode->next) { 3033 if(memcmp(buf, &inode->buf, sizeof(struct stat)) == 0) { 3034 inode->nlink ++; 3035 return inode; 3036 } 3037 } 3038 } 3039 3040 inode = malloc(sizeof(struct inode_info) + bytes); 3041 if(inode == NULL) 3042 MEM_ERROR(); 3043 3044 if(bytes) 3045 memcpy(&inode->symlink, symlink, bytes); 3046 memcpy(&inode->buf, buf, sizeof(struct stat)); 3047 inode->read = FALSE; 3048 inode->root_entry = FALSE; 3049 inode->pseudo_file = pseudo; 3050 inode->pseudo_id = id; 3051 inode->inode = SQUASHFS_INVALID_BLK; 3052 inode->nlink = 1; 3053 inode->inode_number = 0; 3054 3055 /* 3056 * Copy filesystem wide defaults into inode, these filesystem 3057 * wide defaults may be altered on an individual inode basis by 3058 * user specified actions 3059 * 3060 */ 3061 inode->no_fragments = no_fragments; 3062 inode->always_use_fragments = always_use_fragments; 3063 inode->noD = noD; 3064 inode->noF = noF; 3065 3066 inode->next = inode_info[ino_hash]; 3067 inode_info[ino_hash] = inode; 3068 3069 return inode; 3070 } 3071 3072 3073 static inline struct inode_info *lookup_inode2(struct stat *buf, int pseudo, int id) 3074 { 3075 return lookup_inode3(buf, pseudo, id, NULL, 0); 3076 } 3077 3078 3079 static inline struct inode_info *lookup_inode(struct stat *buf) 3080 { 3081 return lookup_inode2(buf, 0, 0); 3082 } 3083 3084 3085 static inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this) 3086 { 3087 if (inode->inode_number == 0) { 3088 inode->inode_number = use_this ? : inode_no ++; 3089 if((inode->buf.st_mode & S_IFMT) == S_IFREG) 3090 progress_bar_size((inode->buf.st_size + block_size - 1) 3091 >> block_log); 3092 } 3093 } 3094 3095 3096 static inline struct dir_ent *create_dir_entry(char *name, char *source_name, 3097 char *nonstandard_pathname, struct dir_info *dir) 3098 { 3099 struct dir_ent *dir_ent = malloc(sizeof(struct dir_ent)); 3100 if(dir_ent == NULL) 3101 MEM_ERROR(); 3102 3103 dir_ent->name = name; 3104 dir_ent->source_name = source_name; 3105 dir_ent->nonstandard_pathname = nonstandard_pathname; 3106 dir_ent->our_dir = dir; 3107 dir_ent->inode = NULL; 3108 dir_ent->next = NULL; 3109 /* ANDROID CHANGES START*/ 3110 #ifdef ANDROID 3111 dir_ent->capabilities = 0; 3112 #endif 3113 /* ANDROID CHANGES END */ 3114 3115 return dir_ent; 3116 } 3117 3118 3119 static inline void add_dir_entry(struct dir_ent *dir_ent, struct dir_info *sub_dir, 3120 struct inode_info *inode_info) 3121 { 3122 struct dir_info *dir = dir_ent->our_dir; 3123 3124 if(sub_dir) 3125 sub_dir->dir_ent = dir_ent; 3126 3127 /* ANDROID CHANGES START*/ 3128 #ifdef ANDROID 3129 if (android_config) { 3130 if (mount_point) { 3131 char *mounted_path; 3132 char *rel_path; 3133 3134 alloc_mounted_path(mount_point, subpathname(dir_ent), &mounted_path); 3135 rel_path = mounted_path; 3136 while (rel_path && *rel_path == '/') 3137 rel_path++; 3138 android_fs_config(fs_config_func, rel_path, &inode_info->buf, target_out_path, &dir_ent->capabilities); 3139 free(mounted_path); 3140 } else { 3141 android_fs_config(fs_config_func, pathname(dir_ent), &inode_info->buf, target_out_path, &dir_ent->capabilities); 3142 } 3143 } 3144 #endif 3145 /* ANDROID CHANGES END */ 3146 3147 dir_ent->inode = inode_info; 3148 dir_ent->dir = sub_dir; 3149 3150 dir_ent->next = dir->list; 3151 dir->list = dir_ent; 3152 dir->count++; 3153 } 3154 3155 static inline void add_dir_entry2(char *name, char *source_name, 3156 char *nonstandard_pathname, struct dir_info *sub_dir, 3157 struct inode_info *inode_info, struct dir_info *dir) 3158 { 3159 struct dir_ent *dir_ent = create_dir_entry(name, source_name, 3160 nonstandard_pathname, dir); 3161 3162 3163 add_dir_entry(dir_ent, sub_dir, inode_info); 3164 } 3165 3166 3167 static inline void free_dir_entry(struct dir_ent *dir_ent) 3168 { 3169 if(dir_ent->name) 3170 free(dir_ent->name); 3171 3172 if(dir_ent->source_name) 3173 free(dir_ent->source_name); 3174 3175 if(dir_ent->nonstandard_pathname) 3176 free(dir_ent->nonstandard_pathname); 3177 3178 /* if this entry has been associated with an inode, then we need 3179 * to update the inode nlink count. Orphaned inodes are harmless, and 3180 * is easier to leave them than go to the bother of deleting them */ 3181 if(dir_ent->inode && !dir_ent->inode->root_entry) 3182 dir_ent->inode->nlink --; 3183 3184 free(dir_ent); 3185 } 3186 3187 3188 static inline void add_excluded(struct dir_info *dir) 3189 { 3190 dir->excluded ++; 3191 } 3192 3193 3194 void dir_scan(squashfs_inode *inode, char *pathname, 3195 struct dir_ent *(_readdir)(struct dir_info *), int progress) 3196 { 3197 struct stat buf; 3198 struct dir_ent *dir_ent; 3199 /* ANDROID CHANGES START*/ 3200 #ifdef ANDROID 3201 uint64_t caps = 0; 3202 #endif 3203 /* ANDROID CHANGES END */ 3204 3205 root_dir = dir_scan1(pathname, "", paths, _readdir, 1); 3206 if(root_dir == NULL) 3207 return; 3208 3209 /* Create root directory dir_ent and associated inode, and connect 3210 * it to the root directory dir_info structure */ 3211 dir_ent = create_dir_entry("", NULL, pathname, 3212 scan1_opendir("", "", 0)); 3213 3214 if(pathname[0] == '\0') { 3215 /* 3216 * dummy top level directory, if multiple sources specified on 3217 * command line 3218 */ 3219 memset(&buf, 0, sizeof(buf)); 3220 buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR; 3221 buf.st_uid = getuid(); 3222 buf.st_gid = getgid(); 3223 buf.st_mtime = time(NULL); 3224 buf.st_dev = 0; 3225 buf.st_ino = 0; 3226 dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0); 3227 } else { 3228 if(lstat(pathname, &buf) == -1) 3229 /* source directory has disappeared? */ 3230 BAD_ERROR("Cannot stat source directory %s because %s\n", 3231 pathname, strerror(errno)); 3232 /* ANDROID CHANGES START*/ 3233 #ifdef ANDROID 3234 buf.st_mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; // root mode 3235 buf.st_uid = 0; 3236 buf.st_gid = 0; 3237 buf.st_mtime = time(NULL); 3238 buf.st_dev = 0; 3239 buf.st_ino = 0; 3240 #endif 3241 /* ANDROID CHANGES END */ 3242 dir_ent->inode = lookup_inode(&buf); 3243 } 3244 3245 /* ANDROID CHANGES START*/ 3246 #ifdef ANDROID 3247 dir_ent->capabilities = caps; 3248 #endif 3249 /* ANDROID CHANGES END */ 3250 3251 dir_ent->dir = root_dir; 3252 root_dir->dir_ent = dir_ent; 3253 3254 /* 3255 * Process most actions and any pseudo files 3256 */ 3257 if(actions() || get_pseudo()) 3258 dir_scan2(root_dir, get_pseudo()); 3259 3260 /* 3261 * Process move actions 3262 */ 3263 if(move_actions()) { 3264 dir_scan3(root_dir); 3265 do_move_actions(); 3266 } 3267 3268 /* 3269 * Process prune actions 3270 */ 3271 if(prune_actions()) 3272 dir_scan4(root_dir); 3273 3274 /* 3275 * Process empty actions 3276 */ 3277 if(empty_actions()) 3278 dir_scan5(root_dir); 3279 3280 /* 3281 * Sort directories and compute the inode numbers 3282 */ 3283 dir_scan6(root_dir); 3284 3285 alloc_inode_no(dir_ent->inode, root_inode_number); 3286 3287 eval_actions(root_dir, dir_ent); 3288 3289 if(sorted) 3290 generate_file_priorities(root_dir, 0, 3291 &root_dir->dir_ent->inode->buf); 3292 3293 if(appending) { 3294 sigset_t sigmask; 3295 3296 restore_thread = init_restore_thread(); 3297 sigemptyset(&sigmask); 3298 sigaddset(&sigmask, SIGINT); 3299 sigaddset(&sigmask, SIGTERM); 3300 sigaddset(&sigmask, SIGUSR1); 3301 if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1) 3302 BAD_ERROR("Failed to set signal mask\n"); 3303 write_destination(fd, SQUASHFS_START, 4, "\0\0\0\0"); 3304 } 3305 3306 queue_put(to_reader, root_dir); 3307 3308 set_progressbar_state(progress); 3309 3310 if(sorted) 3311 sort_files_and_write(root_dir); 3312 3313 dir_scan7(inode, root_dir); 3314 dir_ent->inode->inode = *inode; 3315 dir_ent->inode->type = SQUASHFS_DIR_TYPE; 3316 } 3317 3318 3319 /* 3320 * dir_scan1 routines... 3321 * These scan the source directories into memory for processing. 3322 * Exclude actions are processed here (in contrast to the other actions) 3323 * because they affect what is scanned. 3324 */ 3325 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth) 3326 { 3327 struct dir_info *dir; 3328 3329 dir = malloc(sizeof(struct dir_info)); 3330 if(dir == NULL) 3331 MEM_ERROR(); 3332 3333 if(pathname[0] != '\0') { 3334 dir->linuxdir = opendir(pathname); 3335 if(dir->linuxdir == NULL) { 3336 free(dir); 3337 return NULL; 3338 } 3339 } 3340 3341 dir->pathname = strdup(pathname); 3342 dir->subpath = strdup(subpath); 3343 dir->count = 0; 3344 dir->directory_count = 0; 3345 dir->dir_is_ldir = TRUE; 3346 dir->list = NULL; 3347 dir->depth = depth; 3348 dir->excluded = 0; 3349 3350 return dir; 3351 } 3352 3353 3354 struct dir_ent *scan1_encomp_readdir(struct dir_info *dir) 3355 { 3356 static int index = 0; 3357 3358 if(dir->count < old_root_entries) { 3359 int i; 3360 3361 for(i = 0; i < old_root_entries; i++) { 3362 if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) 3363 dir->directory_count ++; 3364 add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, 3365 &old_root_entry[i].inode, dir); 3366 } 3367 } 3368 3369 while(index < source) { 3370 char *basename = NULL; 3371 char *dir_name = getbase(source_path[index]); 3372 int pass = 1, res; 3373 3374 if(dir_name == NULL) { 3375 ERROR_START("Bad source directory %s", 3376 source_path[index]); 3377 ERROR_EXIT(" - skipping ...\n"); 3378 index ++; 3379 continue; 3380 } 3381 dir_name = strdup(dir_name); 3382 for(;;) { 3383 struct dir_ent *dir_ent = dir->list; 3384 3385 for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; 3386 dir_ent = dir_ent->next); 3387 if(dir_ent == NULL) 3388 break; 3389 ERROR("Source directory entry %s already used! - trying" 3390 " ", dir_name); 3391 if(pass == 1) 3392 basename = dir_name; 3393 else 3394 free(dir_name); 3395 res = asprintf(&dir_name, "%s_%d", basename, pass++); 3396 if(res == -1) 3397 BAD_ERROR("asprintf failed in " 3398 "scan1_encomp_readdir\n"); 3399 ERROR("%s\n", dir_name); 3400 } 3401 return create_dir_entry(dir_name, basename, 3402 strdup(source_path[index ++]), dir); 3403 } 3404 return NULL; 3405 } 3406 3407 3408 struct dir_ent *scan1_single_readdir(struct dir_info *dir) 3409 { 3410 struct dirent *d_name; 3411 int i; 3412 3413 if(dir->count < old_root_entries) { 3414 for(i = 0; i < old_root_entries; i++) { 3415 if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) 3416 dir->directory_count ++; 3417 add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, 3418 &old_root_entry[i].inode, dir); 3419 } 3420 } 3421 3422 if((d_name = readdir(dir->linuxdir)) != NULL) { 3423 char *basename = NULL; 3424 char *dir_name = strdup(d_name->d_name); 3425 int pass = 1, res; 3426 3427 for(;;) { 3428 struct dir_ent *dir_ent = dir->list; 3429 3430 for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; 3431 dir_ent = dir_ent->next); 3432 if(dir_ent == NULL) 3433 break; 3434 ERROR("Source directory entry %s already used! - trying" 3435 " ", dir_name); 3436 if (pass == 1) 3437 basename = dir_name; 3438 else 3439 free(dir_name); 3440 res = asprintf(&dir_name, "%s_%d", d_name->d_name, pass++); 3441 if(res == -1) 3442 BAD_ERROR("asprintf failed in " 3443 "scan1_single_readdir\n"); 3444 ERROR("%s\n", dir_name); 3445 } 3446 return create_dir_entry(dir_name, basename, NULL, dir); 3447 } 3448 3449 return NULL; 3450 } 3451 3452 3453 struct dir_ent *scan1_readdir(struct dir_info *dir) 3454 { 3455 struct dirent *d_name = readdir(dir->linuxdir); 3456 3457 return d_name ? 3458 create_dir_entry(strdup(d_name->d_name), NULL, NULL, dir) : 3459 NULL; 3460 } 3461 3462 3463 void scan1_freedir(struct dir_info *dir) 3464 { 3465 if(dir->pathname[0] != '\0') 3466 closedir(dir->linuxdir); 3467 } 3468 3469 3470 struct dir_info *dir_scan1(char *filename, char *subpath, 3471 struct pathnames *paths, 3472 struct dir_ent *(_readdir)(struct dir_info *), int depth) 3473 { 3474 struct dir_info *dir = scan1_opendir(filename, subpath, depth); 3475 struct dir_ent *dir_ent; 3476 3477 if(dir == NULL) { 3478 ERROR_START("Could not open %s", filename); 3479 ERROR_EXIT(", skipping...\n"); 3480 return NULL; 3481 } 3482 3483 while((dir_ent = _readdir(dir))) { 3484 struct dir_info *sub_dir; 3485 struct stat buf; 3486 struct pathnames *new = NULL; 3487 char *filename = pathname(dir_ent); 3488 char *subpath = NULL; 3489 char *dir_name = dir_ent->name; 3490 3491 if(strcmp(dir_name, ".") == 0 || strcmp(dir_name, "..") == 0) { 3492 free_dir_entry(dir_ent); 3493 continue; 3494 } 3495 3496 if(lstat(filename, &buf) == -1) { 3497 ERROR_START("Cannot stat dir/file %s because %s", 3498 filename, strerror(errno)); 3499 ERROR_EXIT(", ignoring\n"); 3500 free_dir_entry(dir_ent); 3501 continue; 3502 } 3503 3504 if((buf.st_mode & S_IFMT) != S_IFREG && 3505 (buf.st_mode & S_IFMT) != S_IFDIR && 3506 (buf.st_mode & S_IFMT) != S_IFLNK && 3507 (buf.st_mode & S_IFMT) != S_IFCHR && 3508 (buf.st_mode & S_IFMT) != S_IFBLK && 3509 (buf.st_mode & S_IFMT) != S_IFIFO && 3510 (buf.st_mode & S_IFMT) != S_IFSOCK) { 3511 ERROR_START("File %s has unrecognised filetype %d", 3512 filename, buf.st_mode & S_IFMT); 3513 ERROR_EXIT(", ignoring\n"); 3514 free_dir_entry(dir_ent); 3515 continue; 3516 } 3517 3518 if((old_exclude && old_excluded(filename, &buf)) || 3519 (!old_exclude && excluded(dir_name, paths, &new))) { 3520 add_excluded(dir); 3521 free_dir_entry(dir_ent); 3522 continue; 3523 } 3524 3525 if(exclude_actions()) { 3526 subpath = subpathname(dir_ent); 3527 3528 if(eval_exclude_actions(dir_name, filename, subpath, 3529 &buf, depth, dir_ent)) { 3530 add_excluded(dir); 3531 free_dir_entry(dir_ent); 3532 continue; 3533 } 3534 } 3535 3536 switch(buf.st_mode & S_IFMT) { 3537 case S_IFDIR: 3538 if(subpath == NULL) 3539 subpath = subpathname(dir_ent); 3540 3541 sub_dir = dir_scan1(filename, subpath, new, 3542 scan1_readdir, depth + 1); 3543 if(sub_dir) { 3544 dir->directory_count ++; 3545 add_dir_entry(dir_ent, sub_dir, 3546 lookup_inode(&buf)); 3547 } else 3548 free_dir_entry(dir_ent); 3549 break; 3550 case S_IFLNK: { 3551 int byte; 3552 static char buff[65536]; /* overflow safe */ 3553 3554 byte = readlink(filename, buff, 65536); 3555 if(byte == -1) { 3556 ERROR_START("Failed to read symlink %s", 3557 filename); 3558 ERROR_EXIT(", ignoring\n"); 3559 } else if(byte == 65536) { 3560 ERROR_START("Symlink %s is greater than 65536 " 3561 "bytes!", filename); 3562 ERROR_EXIT(", ignoring\n"); 3563 } else { 3564 /* readlink doesn't 0 terminate the returned 3565 * path */ 3566 buff[byte] = '\0'; 3567 add_dir_entry(dir_ent, NULL, lookup_inode3(&buf, 3568 0, 0, buff, byte + 1)); 3569 } 3570 break; 3571 } 3572 default: 3573 add_dir_entry(dir_ent, NULL, lookup_inode(&buf)); 3574 } 3575 3576 free(new); 3577 } 3578 3579 scan1_freedir(dir); 3580 3581 return dir; 3582 } 3583 3584 3585 /* 3586 * dir_scan2 routines... 3587 * This processes most actions and any pseudo files 3588 */ 3589 struct dir_ent *scan2_readdir(struct dir_info *dir, struct dir_ent *dir_ent) 3590 { 3591 if (dir_ent == NULL) 3592 dir_ent = dir->list; 3593 else 3594 dir_ent = dir_ent->next; 3595 3596 for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next); 3597 3598 return dir_ent; 3599 } 3600 3601 3602 struct dir_ent *scan2_lookup(struct dir_info *dir, char *name) 3603 { 3604 struct dir_ent *dir_ent = dir->list; 3605 3606 for(; dir_ent && strcmp(dir_ent->name, name) != 0; 3607 dir_ent = dir_ent->next); 3608 3609 return dir_ent; 3610 } 3611 3612 3613 void dir_scan2(struct dir_info *dir, struct pseudo *pseudo) 3614 { 3615 struct dir_ent *dir_ent = NULL; 3616 struct pseudo_entry *pseudo_ent; 3617 struct stat buf; 3618 static int pseudo_ino = 1; 3619 3620 while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) { 3621 struct inode_info *inode_info = dir_ent->inode; 3622 struct stat *buf = &inode_info->buf; 3623 char *name = dir_ent->name; 3624 3625 eval_actions(root_dir, dir_ent); 3626 3627 if((buf->st_mode & S_IFMT) == S_IFDIR) 3628 dir_scan2(dir_ent->dir, pseudo_subdir(name, pseudo)); 3629 } 3630 3631 while((pseudo_ent = pseudo_readdir(pseudo)) != NULL) { 3632 dir_ent = scan2_lookup(dir, pseudo_ent->name); 3633 if(pseudo_ent->dev->type == 'm') { 3634 struct stat *buf; 3635 if(dir_ent == NULL) { 3636 ERROR_START("Pseudo modify file \"%s\" does " 3637 "not exist in source filesystem.", 3638 pseudo_ent->pathname); 3639 ERROR_EXIT(" Ignoring.\n"); 3640 continue; 3641 } 3642 if(dir_ent->inode->root_entry) { 3643 ERROR_START("Pseudo modify file \"%s\" is a " 3644 "pre-existing file in the filesystem " 3645 "being appended to. It cannot be "\ 3646 "modified.", pseudo_ent->pathname); 3647 ERROR_EXIT(" Ignoring.\n"); 3648 continue; 3649 } 3650 buf = &dir_ent->inode->buf; 3651 buf->st_mode = (buf->st_mode & S_IFMT) | 3652 pseudo_ent->dev->mode; 3653 buf->st_uid = pseudo_ent->dev->uid; 3654 buf->st_gid = pseudo_ent->dev->gid; 3655 continue; 3656 } 3657 3658 if(dir_ent) { 3659 if(dir_ent->inode->root_entry) { 3660 ERROR_START("Pseudo file \"%s\" is a " 3661 "pre-existing file in the filesystem " 3662 "being appended to.", 3663 pseudo_ent->pathname); 3664 ERROR_EXIT(" Ignoring.\n"); 3665 } else { 3666 ERROR_START("Pseudo file \"%s\" exists in " 3667 "source filesystem \"%s\".", 3668 pseudo_ent->pathname, 3669 pathname(dir_ent)); 3670 ERROR_EXIT("\nIgnoring, exclude it (-e/-ef) to " 3671 "override.\n"); 3672 } 3673 continue; 3674 } 3675 3676 memset(&buf, 0, sizeof(buf)); 3677 buf.st_mode = pseudo_ent->dev->mode; 3678 buf.st_uid = pseudo_ent->dev->uid; 3679 buf.st_gid = pseudo_ent->dev->gid; 3680 buf.st_rdev = makedev(pseudo_ent->dev->major, 3681 pseudo_ent->dev->minor); 3682 buf.st_mtime = time(NULL); 3683 buf.st_ino = pseudo_ino ++; 3684 3685 if(pseudo_ent->dev->type == 'd') { 3686 struct dir_ent *dir_ent = 3687 create_dir_entry(pseudo_ent->name, NULL, 3688 pseudo_ent->pathname, dir); 3689 char *subpath = strdup(subpathname(dir_ent)); 3690 struct dir_info *sub_dir = scan1_opendir("", subpath, 3691 dir->depth + 1); 3692 if(sub_dir == NULL) { 3693 ERROR_START("Could not create pseudo directory " 3694 "\"%s\"", pseudo_ent->pathname); 3695 ERROR_EXIT(", skipping...\n"); 3696 free(subpath); 3697 pseudo_ino --; 3698 continue; 3699 } 3700 dir_scan2(sub_dir, pseudo_ent->pseudo); 3701 dir->directory_count ++; 3702 add_dir_entry(dir_ent, sub_dir, 3703 lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0)); 3704 } else if(pseudo_ent->dev->type == 'f') { 3705 add_dir_entry2(pseudo_ent->name, NULL, 3706 pseudo_ent->pathname, NULL, 3707 lookup_inode2(&buf, PSEUDO_FILE_PROCESS, 3708 pseudo_ent->dev->pseudo_id), dir); 3709 } else { 3710 add_dir_entry2(pseudo_ent->name, NULL, 3711 pseudo_ent->pathname, NULL, 3712 lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0), dir); 3713 } 3714 } 3715 } 3716 3717 3718 /* 3719 * dir_scan3 routines... 3720 * This processes the move action 3721 */ 3722 void dir_scan3(struct dir_info *dir) 3723 { 3724 struct dir_ent *dir_ent = NULL; 3725 3726 while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) { 3727 3728 eval_move_actions(root_dir, dir_ent); 3729 3730 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3731 dir_scan3(dir_ent->dir); 3732 } 3733 } 3734 3735 3736 /* 3737 * dir_scan4 routines... 3738 * This processes the prune action. This action is designed to do fine 3739 * grained tuning of the in-core directory structure after the exclude, 3740 * move and pseudo actions have been performed. This allows complex 3741 * tests to be performed which are impossible at exclude time (i.e. 3742 * tests which rely on the in-core directory structure) 3743 */ 3744 void free_dir(struct dir_info *dir) 3745 { 3746 struct dir_ent *dir_ent = dir->list; 3747 3748 while(dir_ent) { 3749 struct dir_ent *tmp = dir_ent; 3750 3751 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3752 free_dir(dir_ent->dir); 3753 3754 dir_ent = dir_ent->next; 3755 free_dir_entry(tmp); 3756 } 3757 3758 free(dir->pathname); 3759 free(dir->subpath); 3760 free(dir); 3761 } 3762 3763 3764 void dir_scan4(struct dir_info *dir) 3765 { 3766 struct dir_ent *dir_ent = dir->list, *prev = NULL; 3767 3768 while(dir_ent) { 3769 if(dir_ent->inode->root_entry) { 3770 prev = dir_ent; 3771 dir_ent = dir_ent->next; 3772 continue; 3773 } 3774 3775 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3776 dir_scan4(dir_ent->dir); 3777 3778 if(eval_prune_actions(root_dir, dir_ent)) { 3779 struct dir_ent *tmp = dir_ent; 3780 3781 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { 3782 free_dir(dir_ent->dir); 3783 dir->directory_count --; 3784 } 3785 3786 dir->count --; 3787 3788 /* remove dir_ent from list */ 3789 dir_ent = dir_ent->next; 3790 if(prev) 3791 prev->next = dir_ent; 3792 else 3793 dir->list = dir_ent; 3794 3795 /* free it */ 3796 free_dir_entry(tmp); 3797 3798 add_excluded(dir); 3799 continue; 3800 } 3801 3802 prev = dir_ent; 3803 dir_ent = dir_ent->next; 3804 } 3805 } 3806 3807 3808 /* 3809 * dir_scan5 routines... 3810 * This processes the empty action. This action has to be processed after 3811 * all other actions because the previous exclude and move actions and the 3812 * pseudo actions affect whether a directory is empty 3813 */ 3814 void dir_scan5(struct dir_info *dir) 3815 { 3816 struct dir_ent *dir_ent = dir->list, *prev = NULL; 3817 3818 while(dir_ent) { 3819 if(dir_ent->inode->root_entry) { 3820 prev = dir_ent; 3821 dir_ent = dir_ent->next; 3822 continue; 3823 } 3824 3825 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { 3826 dir_scan5(dir_ent->dir); 3827 3828 if(eval_empty_actions(root_dir, dir_ent)) { 3829 struct dir_ent *tmp = dir_ent; 3830 3831 /* 3832 * delete sub-directory, this is by definition 3833 * empty 3834 */ 3835 free(dir_ent->dir->pathname); 3836 free(dir_ent->dir->subpath); 3837 free(dir_ent->dir); 3838 3839 /* remove dir_ent from list */ 3840 dir_ent = dir_ent->next; 3841 if(prev) 3842 prev->next = dir_ent; 3843 else 3844 dir->list = dir_ent; 3845 3846 /* free it */ 3847 free_dir_entry(tmp); 3848 3849 /* update counts */ 3850 dir->directory_count --; 3851 dir->count --; 3852 add_excluded(dir); 3853 continue; 3854 } 3855 } 3856 3857 prev = dir_ent; 3858 dir_ent = dir_ent->next; 3859 } 3860 } 3861 3862 3863 /* 3864 * dir_scan6 routines... 3865 * This sorts every directory and computes the inode numbers 3866 */ 3867 3868 /* 3869 * Bottom up linked list merge sort. 3870 * 3871 * Qsort and other O(n log n) algorithms work well with arrays but not 3872 * linked lists. Merge sort another O(n log n) sort algorithm on the other hand 3873 * is not ideal for arrays (as it needs an additonal n storage locations 3874 * as sorting is not done in place), but it is ideal for linked lists because 3875 * it doesn't require any extra storage, 3876 */ 3877 void sort_directory(struct dir_info *dir) 3878 { 3879 struct dir_ent *cur, *l1, *l2, *next; 3880 int len1, len2, stride = 1; 3881 3882 if(dir->list == NULL || dir->count < 2) 3883 return; 3884 3885 /* 3886 * We can consider our linked-list to be made up of stride length 3887 * sublists. Eacn iteration around this loop merges adjacent 3888 * stride length sublists into larger 2*stride sublists. We stop 3889 * when stride becomes equal to the entire list. 3890 * 3891 * Initially stride = 1 (by definition a sublist of 1 is sorted), and 3892 * these 1 element sublists are merged into 2 element sublists, which 3893 * are then merged into 4 element sublists and so on. 3894 */ 3895 do { 3896 l2 = dir->list; /* head of current linked list */ 3897 cur = NULL; /* empty output list */ 3898 3899 /* 3900 * Iterate through the linked list, merging adjacent sublists. 3901 * On each interation l2 points to the next sublist pair to be 3902 * merged (if there's only one sublist left this is simply added 3903 * to the output list) 3904 */ 3905 while(l2) { 3906 l1 = l2; 3907 for(len1 = 0; l2 && len1 < stride; len1 ++, l2 = l2->next); 3908 len2 = stride; 3909 3910 /* 3911 * l1 points to first sublist. 3912 * l2 points to second sublist. 3913 * Merge them onto the output list 3914 */ 3915 while(len1 && l2 && len2) { 3916 if(strcmp(l1->name, l2->name) <= 0) { 3917 next = l1; 3918 l1 = l1->next; 3919 len1 --; 3920 } else { 3921 next = l2; 3922 l2 = l2->next; 3923 len2 --; 3924 } 3925 3926 if(cur) { 3927 cur->next = next; 3928 cur = next; 3929 } else 3930 dir->list = cur = next; 3931 } 3932 /* 3933 * One sublist is now empty, copy the other one onto the 3934 * output list 3935 */ 3936 for(; len1; len1 --, l1 = l1->next) { 3937 if(cur) { 3938 cur->next = l1; 3939 cur = l1; 3940 } else 3941 dir->list = cur = l1; 3942 } 3943 for(; l2 && len2; len2 --, l2 = l2->next) { 3944 if(cur) { 3945 cur->next = l2; 3946 cur = l2; 3947 } else 3948 dir->list = cur = l2; 3949 } 3950 } 3951 cur->next = NULL; 3952 stride = stride << 1; 3953 } while(stride < dir->count); 3954 } 3955 3956 3957 void dir_scan6(struct dir_info *dir) 3958 { 3959 struct dir_ent *dir_ent; 3960 unsigned int byte_count = 0; 3961 3962 sort_directory(dir); 3963 3964 for(dir_ent = dir->list; dir_ent; dir_ent = dir_ent->next) { 3965 byte_count += strlen(dir_ent->name) + 3966 sizeof(struct squashfs_dir_entry); 3967 3968 if(dir_ent->inode->root_entry) 3969 continue; 3970 3971 alloc_inode_no(dir_ent->inode, 0); 3972 3973 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3974 dir_scan6(dir_ent->dir); 3975 } 3976 3977 if((dir->count < 257 && byte_count < SQUASHFS_METADATA_SIZE)) 3978 dir->dir_is_ldir = FALSE; 3979 } 3980 3981 3982 /* 3983 * dir_scan6 routines... 3984 * This generates the filesystem metadata and writes it out to the destination 3985 */ 3986 void scan7_init_dir(struct directory *dir) 3987 { 3988 dir->buff = malloc(SQUASHFS_METADATA_SIZE); 3989 if(dir->buff == NULL) 3990 MEM_ERROR(); 3991 3992 dir->size = SQUASHFS_METADATA_SIZE; 3993 dir->p = dir->index_count_p = dir->buff; 3994 dir->entry_count = 256; 3995 dir->entry_count_p = NULL; 3996 dir->index = NULL; 3997 dir->i_count = dir->i_size = 0; 3998 } 3999 4000 4001 struct dir_ent *scan7_readdir(struct directory *dir, struct dir_info *dir_info, 4002 struct dir_ent *dir_ent) 4003 { 4004 if (dir_ent == NULL) 4005 dir_ent = dir_info->list; 4006 else 4007 dir_ent = dir_ent->next; 4008 4009 for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next) 4010 add_dir(dir_ent->inode->inode, dir_ent->inode->inode_number, 4011 dir_ent->name, dir_ent->inode->type, dir); 4012 4013 return dir_ent; 4014 } 4015 4016 4017 void scan7_freedir(struct directory *dir) 4018 { 4019 if(dir->index) 4020 free(dir->index); 4021 free(dir->buff); 4022 } 4023 4024 4025 void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info) 4026 { 4027 int squashfs_type; 4028 int duplicate_file; 4029 struct directory dir; 4030 struct dir_ent *dir_ent = NULL; 4031 4032 scan7_init_dir(&dir); 4033 4034 while((dir_ent = scan7_readdir(&dir, dir_info, dir_ent)) != NULL) { 4035 struct stat *buf = &dir_ent->inode->buf; 4036 4037 update_info(dir_ent); 4038 4039 if(dir_ent->inode->inode == SQUASHFS_INVALID_BLK) { 4040 switch(buf->st_mode & S_IFMT) { 4041 case S_IFREG: 4042 squashfs_type = SQUASHFS_FILE_TYPE; 4043 write_file(inode, dir_ent, 4044 &duplicate_file); 4045 INFO("file %s, uncompressed size %lld " 4046 "bytes %s\n", 4047 subpathname(dir_ent), 4048 (long long) buf->st_size, 4049 duplicate_file ? "DUPLICATE" : 4050 ""); 4051 break; 4052 4053 case S_IFDIR: 4054 squashfs_type = SQUASHFS_DIR_TYPE; 4055 dir_scan7(inode, dir_ent->dir); 4056 break; 4057 4058 case S_IFLNK: 4059 squashfs_type = SQUASHFS_SYMLINK_TYPE; 4060 create_inode(inode, NULL, dir_ent, 4061 squashfs_type, 0, 0, 0, NULL, 4062 NULL, NULL, 0); 4063 INFO("symbolic link %s inode 0x%llx\n", 4064 subpathname(dir_ent), *inode); 4065 sym_count ++; 4066 break; 4067 4068 case S_IFCHR: 4069 squashfs_type = SQUASHFS_CHRDEV_TYPE; 4070 create_inode(inode, NULL, dir_ent, 4071 squashfs_type, 0, 0, 0, NULL, 4072 NULL, NULL, 0); 4073 INFO("character device %s inode 0x%llx" 4074 "\n", subpathname(dir_ent), 4075 *inode); 4076 dev_count ++; 4077 break; 4078 4079 case S_IFBLK: 4080 squashfs_type = SQUASHFS_BLKDEV_TYPE; 4081 create_inode(inode, NULL, dir_ent, 4082 squashfs_type, 0, 0, 0, NULL, 4083 NULL, NULL, 0); 4084 INFO("block device %s inode 0x%llx\n", 4085 subpathname(dir_ent), *inode); 4086 dev_count ++; 4087 break; 4088 4089 case S_IFIFO: 4090 squashfs_type = SQUASHFS_FIFO_TYPE; 4091 create_inode(inode, NULL, dir_ent, 4092 squashfs_type, 0, 0, 0, NULL, 4093 NULL, NULL, 0); 4094 INFO("fifo %s inode 0x%llx\n", 4095 subpathname(dir_ent), *inode); 4096 fifo_count ++; 4097 break; 4098 4099 case S_IFSOCK: 4100 squashfs_type = SQUASHFS_SOCKET_TYPE; 4101 create_inode(inode, NULL, dir_ent, 4102 squashfs_type, 0, 0, 0, NULL, 4103 NULL, NULL, 0); 4104 INFO("unix domain socket %s inode " 4105 "0x%llx\n", 4106 subpathname(dir_ent), *inode); 4107 sock_count ++; 4108 break; 4109 4110 default: 4111 BAD_ERROR("%s unrecognised file type, " 4112 "mode is %x\n", 4113 subpathname(dir_ent), 4114 buf->st_mode); 4115 } 4116 dir_ent->inode->inode = *inode; 4117 dir_ent->inode->type = squashfs_type; 4118 } else { 4119 *inode = dir_ent->inode->inode; 4120 squashfs_type = dir_ent->inode->type; 4121 switch(squashfs_type) { 4122 case SQUASHFS_FILE_TYPE: 4123 if(!sorted) 4124 INFO("file %s, uncompressed " 4125 "size %lld bytes LINK" 4126 "\n", 4127 subpathname(dir_ent), 4128 (long long) 4129 buf->st_size); 4130 break; 4131 case SQUASHFS_SYMLINK_TYPE: 4132 INFO("symbolic link %s inode 0x%llx " 4133 "LINK\n", subpathname(dir_ent), 4134 *inode); 4135 break; 4136 case SQUASHFS_CHRDEV_TYPE: 4137 INFO("character device %s inode 0x%llx " 4138 "LINK\n", subpathname(dir_ent), 4139 *inode); 4140 break; 4141 case SQUASHFS_BLKDEV_TYPE: 4142 INFO("block device %s inode 0x%llx " 4143 "LINK\n", subpathname(dir_ent), 4144 *inode); 4145 break; 4146 case SQUASHFS_FIFO_TYPE: 4147 INFO("fifo %s inode 0x%llx LINK\n", 4148 subpathname(dir_ent), *inode); 4149 break; 4150 case SQUASHFS_SOCKET_TYPE: 4151 INFO("unix domain socket %s inode " 4152 "0x%llx LINK\n", 4153 subpathname(dir_ent), *inode); 4154 break; 4155 } 4156 } 4157 4158 add_dir(*inode, get_inode_no(dir_ent->inode), dir_ent->name, 4159 squashfs_type, &dir); 4160 } 4161 4162 write_dir(inode, dir_info, &dir); 4163 INFO("directory %s inode 0x%llx\n", subpathname(dir_info->dir_ent), 4164 *inode); 4165 4166 scan7_freedir(&dir); 4167 } 4168 4169 4170 unsigned int slog(unsigned int block) 4171 { 4172 int i; 4173 4174 for(i = 12; i <= 20; i++) 4175 if(block == (1 << i)) 4176 return i; 4177 return 0; 4178 } 4179 4180 4181 int old_excluded(char *filename, struct stat *buf) 4182 { 4183 int i; 4184 4185 for(i = 0; i < exclude; i++) 4186 if((exclude_paths[i].st_dev == buf->st_dev) && 4187 (exclude_paths[i].st_ino == buf->st_ino)) 4188 return TRUE; 4189 return FALSE; 4190 } 4191 4192 4193 #define ADD_ENTRY(buf) \ 4194 if(exclude % EXCLUDE_SIZE == 0) { \ 4195 exclude_paths = realloc(exclude_paths, (exclude + EXCLUDE_SIZE) \ 4196 * sizeof(struct exclude_info)); \ 4197 if(exclude_paths == NULL) \ 4198 MEM_ERROR(); \ 4199 } \ 4200 exclude_paths[exclude].st_dev = buf.st_dev; \ 4201 exclude_paths[exclude++].st_ino = buf.st_ino; 4202 int old_add_exclude(char *path) 4203 { 4204 int i; 4205 char *filename; 4206 struct stat buf; 4207 4208 if(path[0] == '/' || strncmp(path, "./", 2) == 0 || 4209 strncmp(path, "../", 3) == 0) { 4210 if(lstat(path, &buf) == -1) { 4211 ERROR_START("Cannot stat exclude dir/file %s because " 4212 "%s", path, strerror(errno)); 4213 ERROR_EXIT(", ignoring\n"); 4214 return TRUE; 4215 } 4216 ADD_ENTRY(buf); 4217 return TRUE; 4218 } 4219 4220 for(i = 0; i < source; i++) { 4221 int res = asprintf(&filename, "%s/%s", source_path[i], path); 4222 if(res == -1) 4223 BAD_ERROR("asprintf failed in old_add_exclude\n"); 4224 if(lstat(filename, &buf) == -1) { 4225 if(!(errno == ENOENT || errno == ENOTDIR)) { 4226 ERROR_START("Cannot stat exclude dir/file %s " 4227 "because %s", filename, strerror(errno)); 4228 ERROR_EXIT(", ignoring\n"); 4229 } 4230 free(filename); 4231 continue; 4232 } 4233 free(filename); 4234 ADD_ENTRY(buf); 4235 } 4236 return TRUE; 4237 } 4238 4239 4240 void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, 4241 int type) 4242 { 4243 old_root_entry = realloc(old_root_entry, 4244 sizeof(struct old_root_entry_info) * (old_root_entries + 1)); 4245 if(old_root_entry == NULL) 4246 MEM_ERROR(); 4247 4248 old_root_entry[old_root_entries].name = strdup(name); 4249 old_root_entry[old_root_entries].inode.inode = inode; 4250 old_root_entry[old_root_entries].inode.inode_number = inode_number; 4251 old_root_entry[old_root_entries].inode.type = type; 4252 old_root_entry[old_root_entries++].inode.root_entry = TRUE; 4253 } 4254 4255 4256 void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, 4257 int freelst, char *destination_file) 4258 { 4259 int i; 4260 sigset_t sigmask, old_mask; 4261 int total_mem = readq; 4262 int reader_size; 4263 int fragment_size; 4264 int fwriter_size; 4265 /* 4266 * bwriter_size is global because it is needed in 4267 * write_file_blocks_dup() 4268 */ 4269 4270 /* 4271 * Never allow the total size of the queues to be larger than 4272 * physical memory 4273 * 4274 * When adding together the possibly user supplied values, make 4275 * sure they've not been deliberately contrived to overflow an int 4276 */ 4277 if(add_overflow(total_mem, fragq)) 4278 BAD_ERROR("Queue sizes rediculously too large\n"); 4279 total_mem += fragq; 4280 if(add_overflow(total_mem, bwriteq)) 4281 BAD_ERROR("Queue sizes rediculously too large\n"); 4282 total_mem += bwriteq; 4283 if(add_overflow(total_mem, fwriteq)) 4284 BAD_ERROR("Queue sizes rediculously too large\n"); 4285 total_mem += fwriteq; 4286 4287 check_usable_phys_mem(total_mem); 4288 4289 /* 4290 * convert from queue size in Mbytes to queue size in 4291 * blocks. 4292 * 4293 * This isn't going to overflow an int unless there exists 4294 * systems with more than 8 Petabytes of RAM! 4295 */ 4296 reader_size = readq << (20 - block_log); 4297 fragment_size = fragq << (20 - block_log); 4298 bwriter_size = bwriteq << (20 - block_log); 4299 fwriter_size = fwriteq << (20 - block_log); 4300 4301 /* 4302 * setup signal handlers for the main thread, these cleanup 4303 * deleting the destination file, if appending the 4304 * handlers for SIGTERM and SIGINT will be replaced with handlers 4305 * allowing the user to press ^C twice to restore the existing 4306 * filesystem. 4307 * 4308 * SIGUSR1 is an internal signal, which is used by the sub-threads 4309 * to tell the main thread to terminate, deleting the destination file, 4310 * or if necessary restoring the filesystem on appending 4311 */ 4312 signal(SIGTERM, sighandler); 4313 signal(SIGINT, sighandler); 4314 signal(SIGUSR1, sighandler); 4315 4316 /* block SIGQUIT and SIGHUP, these are handled by the info thread */ 4317 sigemptyset(&sigmask); 4318 sigaddset(&sigmask, SIGQUIT); 4319 sigaddset(&sigmask, SIGHUP); 4320 sigaddset(&sigmask, SIGALRM); 4321 if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1) 4322 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4323 4324 /* 4325 * temporarily block these signals, so the created sub-threads 4326 * will ignore them, ensuring the main thread handles them 4327 */ 4328 sigemptyset(&sigmask); 4329 sigaddset(&sigmask, SIGINT); 4330 sigaddset(&sigmask, SIGTERM); 4331 sigaddset(&sigmask, SIGUSR1); 4332 if(pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask) == -1) 4333 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4334 4335 if(processors == -1) { 4336 #ifndef linux 4337 int mib[2]; 4338 size_t len = sizeof(processors); 4339 4340 mib[0] = CTL_HW; 4341 #ifdef HW_AVAILCPU 4342 mib[1] = HW_AVAILCPU; 4343 #else 4344 mib[1] = HW_NCPU; 4345 #endif 4346 4347 if(sysctl(mib, 2, &processors, &len, NULL, 0) == -1) { 4348 ERROR_START("Failed to get number of available " 4349 "processors."); 4350 ERROR_EXIT(" Defaulting to 1\n"); 4351 processors = 1; 4352 } 4353 #else 4354 processors = sysconf(_SC_NPROCESSORS_ONLN); 4355 #endif 4356 } 4357 4358 if(multiply_overflow(processors, 3) || 4359 multiply_overflow(processors * 3, sizeof(pthread_t))) 4360 BAD_ERROR("Processors too large\n"); 4361 4362 deflator_thread = malloc(processors * 3 * sizeof(pthread_t)); 4363 if(deflator_thread == NULL) 4364 MEM_ERROR(); 4365 4366 frag_deflator_thread = &deflator_thread[processors]; 4367 frag_thread = &frag_deflator_thread[processors]; 4368 4369 to_reader = queue_init(1); 4370 to_deflate = queue_init(reader_size); 4371 to_process_frag = queue_init(reader_size); 4372 to_writer = queue_init(bwriter_size + fwriter_size); 4373 from_writer = queue_init(1); 4374 to_frag = queue_init(fragment_size); 4375 locked_fragment = queue_init(fragment_size); 4376 to_main = seq_queue_init(); 4377 reader_buffer = cache_init(block_size, reader_size, 0, 0); 4378 bwriter_buffer = cache_init(block_size, bwriter_size, 1, freelst); 4379 fwriter_buffer = cache_init(block_size, fwriter_size, 1, freelst); 4380 fragment_buffer = cache_init(block_size, fragment_size, 1, 0); 4381 reserve_cache = cache_init(block_size, processors + 1, 1, 0); 4382 pthread_create(&reader_thread, NULL, reader, NULL); 4383 pthread_create(&writer_thread, NULL, writer, NULL); 4384 init_progress_bar(); 4385 init_info(); 4386 4387 for(i = 0; i < processors; i++) { 4388 if(pthread_create(&deflator_thread[i], NULL, deflator, NULL)) 4389 BAD_ERROR("Failed to create thread\n"); 4390 if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator, 4391 NULL) != 0) 4392 BAD_ERROR("Failed to create thread\n"); 4393 if(pthread_create(&frag_thread[i], NULL, frag_thrd, 4394 (void *) destination_file) != 0) 4395 BAD_ERROR("Failed to create thread\n"); 4396 } 4397 4398 main_thread = pthread_self(); 4399 4400 printf("Parallel mksquashfs: Using %d processor%s\n", processors, 4401 processors == 1 ? "" : "s"); 4402 4403 /* Restore the signal mask for the main thread */ 4404 if(pthread_sigmask(SIG_SETMASK, &old_mask, NULL) == -1) 4405 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4406 } 4407 4408 4409 long long write_inode_lookup_table() 4410 { 4411 int i, inode_number, lookup_bytes = SQUASHFS_LOOKUP_BYTES(inode_count); 4412 void *it; 4413 4414 if(inode_count == sinode_count) 4415 goto skip_inode_hash_table; 4416 4417 it = realloc(inode_lookup_table, lookup_bytes); 4418 if(it == NULL) 4419 MEM_ERROR(); 4420 inode_lookup_table = it; 4421 4422 for(i = 0; i < INODE_HASH_SIZE; i ++) { 4423 struct inode_info *inode; 4424 4425 for(inode = inode_info[i]; inode; inode = inode->next) { 4426 4427 inode_number = get_inode_no(inode); 4428 4429 /* The empty action will produce orphaned inode 4430 * entries in the inode_info[] table. These 4431 * entries because they are orphaned will not be 4432 * allocated an inode number in dir_scan5(), so 4433 * skip any entries with the default dummy inode 4434 * number of 0 */ 4435 if(inode_number == 0) 4436 continue; 4437 4438 SQUASHFS_SWAP_LONG_LONGS(&inode->inode, 4439 &inode_lookup_table[inode_number - 1], 1); 4440 4441 } 4442 } 4443 4444 skip_inode_hash_table: 4445 return generic_write_table(lookup_bytes, inode_lookup_table, 0, NULL, 4446 noI); 4447 } 4448 4449 4450 char *get_component(char *target, char **targname) 4451 { 4452 char *start; 4453 4454 while(*target == '/') 4455 target ++; 4456 4457 start = target; 4458 while(*target != '/' && *target != '\0') 4459 target ++; 4460 4461 *targname = strndup(start, target - start); 4462 4463 while(*target == '/') 4464 target ++; 4465 4466 return target; 4467 } 4468 4469 4470 void free_path(struct pathname *paths) 4471 { 4472 int i; 4473 4474 for(i = 0; i < paths->names; i++) { 4475 if(paths->name[i].paths) 4476 free_path(paths->name[i].paths); 4477 free(paths->name[i].name); 4478 if(paths->name[i].preg) { 4479 regfree(paths->name[i].preg); 4480 free(paths->name[i].preg); 4481 } 4482 } 4483 4484 free(paths); 4485 } 4486 4487 4488 struct pathname *add_path(struct pathname *paths, char *target, char *alltarget) 4489 { 4490 char *targname; 4491 int i, error; 4492 4493 target = get_component(target, &targname); 4494 4495 if(paths == NULL) { 4496 paths = malloc(sizeof(struct pathname)); 4497 if(paths == NULL) 4498 MEM_ERROR(); 4499 4500 paths->names = 0; 4501 paths->name = NULL; 4502 } 4503 4504 for(i = 0; i < paths->names; i++) 4505 if(strcmp(paths->name[i].name, targname) == 0) 4506 break; 4507 4508 if(i == paths->names) { 4509 /* allocate new name entry */ 4510 paths->names ++; 4511 paths->name = realloc(paths->name, (i + 1) * 4512 sizeof(struct path_entry)); 4513 if(paths->name == NULL) 4514 MEM_ERROR(); 4515 paths->name[i].name = targname; 4516 paths->name[i].paths = NULL; 4517 if(use_regex) { 4518 paths->name[i].preg = malloc(sizeof(regex_t)); 4519 if(paths->name[i].preg == NULL) 4520 MEM_ERROR(); 4521 error = regcomp(paths->name[i].preg, targname, 4522 REG_EXTENDED|REG_NOSUB); 4523 if(error) { 4524 char str[1024]; /* overflow safe */ 4525 4526 regerror(error, paths->name[i].preg, str, 1024); 4527 BAD_ERROR("invalid regex %s in export %s, " 4528 "because %s\n", targname, alltarget, 4529 str); 4530 } 4531 } else 4532 paths->name[i].preg = NULL; 4533 4534 if(target[0] == '\0') 4535 /* at leaf pathname component */ 4536 paths->name[i].paths = NULL; 4537 else 4538 /* recurse adding child components */ 4539 paths->name[i].paths = add_path(NULL, target, 4540 alltarget); 4541 } else { 4542 /* existing matching entry */ 4543 free(targname); 4544 4545 if(paths->name[i].paths == NULL) { 4546 /* No sub-directory which means this is the leaf 4547 * component of a pre-existing exclude which subsumes 4548 * the exclude currently being added, in which case stop 4549 * adding components */ 4550 } else if(target[0] == '\0') { 4551 /* at leaf pathname component and child components exist 4552 * from more specific excludes, delete as they're 4553 * subsumed by this exclude */ 4554 free_path(paths->name[i].paths); 4555 paths->name[i].paths = NULL; 4556 } else 4557 /* recurse adding child components */ 4558 add_path(paths->name[i].paths, target, alltarget); 4559 } 4560 4561 return paths; 4562 } 4563 4564 4565 void add_exclude(char *target) 4566 { 4567 4568 if(target[0] == '/' || strncmp(target, "./", 2) == 0 || 4569 strncmp(target, "../", 3) == 0) 4570 BAD_ERROR("/, ./ and ../ prefixed excludes not supported with " 4571 "-wildcards or -regex options\n"); 4572 else if(strncmp(target, "... ", 4) == 0) 4573 stickypath = add_path(stickypath, target + 4, target + 4); 4574 else 4575 path = add_path(path, target, target); 4576 } 4577 4578 4579 void display_path(int depth, struct pathname *paths) 4580 { 4581 int i, n; 4582 4583 if(paths == NULL) 4584 return; 4585 4586 for(i = 0; i < paths->names; i++) { 4587 for(n = 0; n < depth; n++) 4588 printf("\t"); 4589 printf("%d: %s\n", depth, paths->name[i].name); 4590 display_path(depth + 1, paths->name[i].paths); 4591 } 4592 } 4593 4594 4595 void display_path2(struct pathname *paths, char *string) 4596 { 4597 int i; 4598 char *path; 4599 4600 if(paths == NULL) { 4601 printf("%s\n", string); 4602 return; 4603 } 4604 4605 for(i = 0; i < paths->names; i++) { 4606 int res = asprintf(&path, "%s/%s", string, paths->name[i].name); 4607 if(res == -1) 4608 BAD_ERROR("asprintf failed in display_path2\n"); 4609 display_path2(paths->name[i].paths, path); 4610 free(path); 4611 } 4612 } 4613 4614 4615 struct pathnames *add_subdir(struct pathnames *paths, struct pathname *path) 4616 { 4617 int count = paths == NULL ? 0 : paths->count; 4618 4619 if(count % PATHS_ALLOC_SIZE == 0) { 4620 paths = realloc(paths, sizeof(struct pathnames) + 4621 (count + PATHS_ALLOC_SIZE) * sizeof(struct pathname *)); 4622 if(paths == NULL) 4623 MEM_ERROR(); 4624 } 4625 4626 paths->path[count] = path; 4627 paths->count = count + 1; 4628 return paths; 4629 } 4630 4631 4632 int excluded_match(char *name, struct pathname *path, struct pathnames **new) 4633 { 4634 int i; 4635 4636 for(i = 0; i < path->names; i++) { 4637 int match = use_regex ? 4638 regexec(path->name[i].preg, name, (size_t) 0, 4639 NULL, 0) == 0 : 4640 fnmatch(path->name[i].name, name, 4641 FNM_PATHNAME|FNM_PERIOD|FNM_EXTMATCH) == 0; 4642 4643 if(match) { 4644 if(path->name[i].paths == NULL || new == NULL) 4645 /* match on a leaf component, any subdirectories 4646 * in the filesystem should be excluded */ 4647 return TRUE; 4648 else 4649 /* match on a non-leaf component, add any 4650 * subdirectories to the new set of 4651 * subdirectories to scan for this name */ 4652 *new = add_subdir(*new, path->name[i].paths); 4653 } 4654 } 4655 4656 return FALSE; 4657 } 4658 4659 4660 int excluded(char *name, struct pathnames *paths, struct pathnames **new) 4661 { 4662 int n; 4663 4664 if(stickypath && excluded_match(name, stickypath, NULL)) 4665 return TRUE; 4666 4667 for(n = 0; paths && n < paths->count; n++) { 4668 int res = excluded_match(name, paths->path[n], new); 4669 if(res) { 4670 free(*new); 4671 *new = NULL; 4672 return TRUE; 4673 } 4674 } 4675 4676 /* 4677 * Either: 4678 * - no matching names found, return empty new search set, or 4679 * - one or more matches with sub-directories found (no leaf matches), 4680 * in which case return new search set. 4681 * 4682 * In either case return FALSE as we don't want to exclude this entry 4683 */ 4684 return FALSE; 4685 } 4686 4687 4688 void process_exclude_file(char *argv) 4689 { 4690 FILE *fd; 4691 char buffer[MAX_LINE + 1]; /* overflow safe */ 4692 char *filename; 4693 4694 fd = fopen(argv, "r"); 4695 if(fd == NULL) 4696 BAD_ERROR("Failed to open exclude file \"%s\" because %s\n", 4697 argv, strerror(errno)); 4698 4699 while(fgets(filename = buffer, MAX_LINE + 1, fd) != NULL) { 4700 int len = strlen(filename); 4701 4702 if(len == MAX_LINE && filename[len - 1] != '\n') 4703 /* line too large */ 4704 BAD_ERROR("Line too long when reading " 4705 "exclude file \"%s\", larger than %d " 4706 "bytes\n", argv, MAX_LINE); 4707 4708 /* 4709 * Remove '\n' terminator if it exists (the last line 4710 * in the file may not be '\n' terminated) 4711 */ 4712 if(len && filename[len - 1] == '\n') 4713 filename[len - 1] = '\0'; 4714 4715 /* Skip any leading whitespace */ 4716 while(isspace(*filename)) 4717 filename ++; 4718 4719 /* if comment line, skip */ 4720 if(*filename == '#') 4721 continue; 4722 4723 /* 4724 * check for initial backslash, to accommodate 4725 * filenames with leading space or leading # character 4726 */ 4727 if(*filename == '\\') 4728 filename ++; 4729 4730 /* if line is now empty after skipping characters, skip it */ 4731 if(*filename == '\0') 4732 continue; 4733 4734 if(old_exclude) 4735 old_add_exclude(filename); 4736 else 4737 add_exclude(filename); 4738 } 4739 4740 if(ferror(fd)) 4741 BAD_ERROR("Reading exclude file \"%s\" failed because %s\n", 4742 argv, strerror(errno)); 4743 4744 fclose(fd); 4745 } 4746 4747 4748 #define RECOVER_ID "Squashfs recovery file v1.0\n" 4749 #define RECOVER_ID_SIZE 28 4750 4751 void write_recovery_data(struct squashfs_super_block *sBlk) 4752 { 4753 int res, recoverfd, bytes = sBlk->bytes_used - sBlk->inode_table_start; 4754 pid_t pid = getpid(); 4755 char *metadata; 4756 char header[] = RECOVER_ID; 4757 4758 if(recover == FALSE) { 4759 printf("No recovery data option specified.\n"); 4760 printf("Skipping saving recovery file.\n\n"); 4761 return; 4762 } 4763 4764 metadata = malloc(bytes); 4765 if(metadata == NULL) 4766 MEM_ERROR(); 4767 4768 res = read_fs_bytes(fd, sBlk->inode_table_start, bytes, metadata); 4769 if(res == 0) { 4770 ERROR("Failed to read append filesystem metadata\n"); 4771 BAD_ERROR("Filesystem corrupted?\n"); 4772 } 4773 4774 res = asprintf(&recovery_file, "squashfs_recovery_%s_%d", 4775 getbase(destination_file), pid); 4776 if(res == -1) 4777 MEM_ERROR(); 4778 4779 recoverfd = open(recovery_file, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU); 4780 if(recoverfd == -1) 4781 BAD_ERROR("Failed to create recovery file, because %s. " 4782 "Aborting\n", strerror(errno)); 4783 4784 if(write_bytes(recoverfd, header, RECOVER_ID_SIZE) == -1) 4785 BAD_ERROR("Failed to write recovery file, because %s\n", 4786 strerror(errno)); 4787 4788 if(write_bytes(recoverfd, sBlk, sizeof(struct squashfs_super_block)) == -1) 4789 BAD_ERROR("Failed to write recovery file, because %s\n", 4790 strerror(errno)); 4791 4792 if(write_bytes(recoverfd, metadata, bytes) == -1) 4793 BAD_ERROR("Failed to write recovery file, because %s\n", 4794 strerror(errno)); 4795 4796 close(recoverfd); 4797 free(metadata); 4798 4799 printf("Recovery file \"%s\" written\n", recovery_file); 4800 printf("If Mksquashfs aborts abnormally (i.e. power failure), run\n"); 4801 printf("mksquashfs dummy %s -recover %s\n", destination_file, 4802 recovery_file); 4803 printf("to restore filesystem\n\n"); 4804 } 4805 4806 4807 void read_recovery_data(char *recovery_file, char *destination_file) 4808 { 4809 int fd, recoverfd, bytes; 4810 struct squashfs_super_block orig_sBlk, sBlk; 4811 char *metadata; 4812 int res; 4813 struct stat buf; 4814 char header[] = RECOVER_ID; 4815 char header2[RECOVER_ID_SIZE]; 4816 4817 recoverfd = open(recovery_file, O_RDONLY); 4818 if(recoverfd == -1) 4819 BAD_ERROR("Failed to open recovery file because %s\n", 4820 strerror(errno)); 4821 4822 if(stat(destination_file, &buf) == -1) 4823 BAD_ERROR("Failed to stat destination file, because %s\n", 4824 strerror(errno)); 4825 4826 fd = open(destination_file, O_RDWR); 4827 if(fd == -1) 4828 BAD_ERROR("Failed to open destination file because %s\n", 4829 strerror(errno)); 4830 4831 res = read_bytes(recoverfd, header2, RECOVER_ID_SIZE); 4832 if(res == -1) 4833 BAD_ERROR("Failed to read recovery file, because %s\n", 4834 strerror(errno)); 4835 if(res < RECOVER_ID_SIZE) 4836 BAD_ERROR("Recovery file appears to be truncated\n"); 4837 if(strncmp(header, header2, RECOVER_ID_SIZE) !=0 ) 4838 BAD_ERROR("Not a recovery file\n"); 4839 4840 res = read_bytes(recoverfd, &sBlk, sizeof(struct squashfs_super_block)); 4841 if(res == -1) 4842 BAD_ERROR("Failed to read recovery file, because %s\n", 4843 strerror(errno)); 4844 if(res < sizeof(struct squashfs_super_block)) 4845 BAD_ERROR("Recovery file appears to be truncated\n"); 4846 4847 res = read_fs_bytes(fd, 0, sizeof(struct squashfs_super_block), &orig_sBlk); 4848 if(res == 0) { 4849 ERROR("Failed to read superblock from output filesystem\n"); 4850 BAD_ERROR("Output filesystem is empty!\n"); 4851 } 4852 4853 if(memcmp(((char *) &sBlk) + 4, ((char *) &orig_sBlk) + 4, 4854 sizeof(struct squashfs_super_block) - 4) != 0) 4855 BAD_ERROR("Recovery file and destination file do not seem to " 4856 "match\n"); 4857 4858 bytes = sBlk.bytes_used - sBlk.inode_table_start; 4859 4860 metadata = malloc(bytes); 4861 if(metadata == NULL) 4862 MEM_ERROR(); 4863 4864 res = read_bytes(recoverfd, metadata, bytes); 4865 if(res == -1) 4866 BAD_ERROR("Failed to read recovery file, because %s\n", 4867 strerror(errno)); 4868 if(res < bytes) 4869 BAD_ERROR("Recovery file appears to be truncated\n"); 4870 4871 write_destination(fd, 0, sizeof(struct squashfs_super_block), &sBlk); 4872 4873 write_destination(fd, sBlk.inode_table_start, bytes, metadata); 4874 4875 close(recoverfd); 4876 close(fd); 4877 4878 printf("Successfully wrote recovery file \"%s\". Exiting\n", 4879 recovery_file); 4880 4881 exit(0); 4882 } 4883 4884 4885 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad) 4886 { 4887 int i; 4888 4889 sBlk->fragments = fragments; 4890 sBlk->no_ids = id_count; 4891 sBlk->inode_table_start = write_inodes(); 4892 sBlk->directory_table_start = write_directories(); 4893 sBlk->fragment_table_start = write_fragment_table(); 4894 sBlk->lookup_table_start = exportable ? write_inode_lookup_table() : 4895 SQUASHFS_INVALID_BLK; 4896 sBlk->id_table_start = write_id_table(); 4897 sBlk->xattr_id_table_start = write_xattrs(); 4898 4899 TRACE("sBlk->inode_table_start 0x%llx\n", sBlk->inode_table_start); 4900 TRACE("sBlk->directory_table_start 0x%llx\n", 4901 sBlk->directory_table_start); 4902 TRACE("sBlk->fragment_table_start 0x%llx\n", sBlk->fragment_table_start); 4903 if(exportable) 4904 TRACE("sBlk->lookup_table_start 0x%llx\n", 4905 sBlk->lookup_table_start); 4906 4907 sBlk->bytes_used = bytes; 4908 4909 sBlk->compression = comp->id; 4910 4911 SQUASHFS_INSWAP_SUPER_BLOCK(sBlk); 4912 write_destination(fd, SQUASHFS_START, sizeof(*sBlk), sBlk); 4913 4914 if(!nopad && (i = bytes & (4096 - 1))) { 4915 char temp[4096] = {0}; 4916 write_destination(fd, bytes, 4096 - i, temp); 4917 } 4918 4919 close(fd); 4920 4921 if(recovery_file) 4922 unlink(recovery_file); 4923 4924 total_bytes += total_inode_bytes + total_directory_bytes + 4925 sizeof(struct squashfs_super_block) + total_xattr_bytes; 4926 4927 printf("\n%sSquashfs %d.%d filesystem, %s compressed, data block size" 4928 " %d\n", exportable ? "Exportable " : "", SQUASHFS_MAJOR, 4929 SQUASHFS_MINOR, comp->name, block_size); 4930 printf("\t%s data, %s metadata, %s fragments, %s xattrs\n", 4931 noD ? "uncompressed" : "compressed", noI ? "uncompressed" : 4932 "compressed", no_fragments ? "no" : noF ? "uncompressed" : 4933 "compressed", no_xattrs ? "no" : noX ? "uncompressed" : 4934 "compressed"); 4935 printf("\tduplicates are %sremoved\n", duplicate_checking ? "" : 4936 "not "); 4937 printf("Filesystem size %.2f Kbytes (%.2f Mbytes)\n", bytes / 1024.0, 4938 bytes / (1024.0 * 1024.0)); 4939 printf("\t%.2f%% of uncompressed filesystem size (%.2f Kbytes)\n", 4940 ((float) bytes / total_bytes) * 100.0, total_bytes / 1024.0); 4941 printf("Inode table size %d bytes (%.2f Kbytes)\n", 4942 inode_bytes, inode_bytes / 1024.0); 4943 printf("\t%.2f%% of uncompressed inode table size (%d bytes)\n", 4944 ((float) inode_bytes / total_inode_bytes) * 100.0, 4945 total_inode_bytes); 4946 printf("Directory table size %d bytes (%.2f Kbytes)\n", 4947 directory_bytes, directory_bytes / 1024.0); 4948 printf("\t%.2f%% of uncompressed directory table size (%d bytes)\n", 4949 ((float) directory_bytes / total_directory_bytes) * 100.0, 4950 total_directory_bytes); 4951 if(total_xattr_bytes) { 4952 printf("Xattr table size %d bytes (%.2f Kbytes)\n", 4953 xattr_bytes, xattr_bytes / 1024.0); 4954 printf("\t%.2f%% of uncompressed xattr table size (%d bytes)\n", 4955 ((float) xattr_bytes / total_xattr_bytes) * 100.0, 4956 total_xattr_bytes); 4957 } 4958 if(duplicate_checking) 4959 printf("Number of duplicate files found %d\n", file_count - 4960 dup_files); 4961 else 4962 printf("No duplicate files removed\n"); 4963 printf("Number of inodes %d\n", inode_count); 4964 printf("Number of files %d\n", file_count); 4965 if(!no_fragments) 4966 printf("Number of fragments %d\n", fragments); 4967 printf("Number of symbolic links %d\n", sym_count); 4968 printf("Number of device nodes %d\n", dev_count); 4969 printf("Number of fifo nodes %d\n", fifo_count); 4970 printf("Number of socket nodes %d\n", sock_count); 4971 printf("Number of directories %d\n", dir_count); 4972 printf("Number of ids (unique uids + gids) %d\n", id_count); 4973 printf("Number of uids %d\n", uid_count); 4974 4975 for(i = 0; i < id_count; i++) { 4976 if(id_table[i]->flags & ISA_UID) { 4977 struct passwd *user = getpwuid(id_table[i]->id); 4978 printf("\t%s (%d)\n", user == NULL ? "unknown" : 4979 user->pw_name, id_table[i]->id); 4980 } 4981 } 4982 4983 printf("Number of gids %d\n", guid_count); 4984 4985 for(i = 0; i < id_count; i++) { 4986 if(id_table[i]->flags & ISA_GID) { 4987 struct group *group = getgrgid(id_table[i]->id); 4988 printf("\t%s (%d)\n", group == NULL ? "unknown" : 4989 group->gr_name, id_table[i]->id); 4990 } 4991 } 4992 } 4993 4994 4995 int parse_numberll(char *start, long long *res, int size) 4996 { 4997 char *end; 4998 long long number; 4999 5000 errno = 0; /* To distinguish success/failure after call */ 5001 5002 number = strtoll(start, &end, 10); 5003 5004 /* 5005 * check for strtoll underflow or overflow in conversion, and other 5006 * errors. 5007 */ 5008 if((errno == ERANGE && (number == LLONG_MIN || number == LLONG_MAX)) || 5009 (errno != 0 && number == 0)) 5010 return 0; 5011 5012 /* reject negative numbers as invalid */ 5013 if(number < 0) 5014 return 0; 5015 5016 if(size) { 5017 /* 5018 * Check for multiplier and trailing junk. 5019 * But first check that a number exists before the 5020 * multiplier 5021 */ 5022 if(end == start) 5023 return 0; 5024 5025 switch(end[0]) { 5026 case 'g': 5027 case 'G': 5028 if(multiply_overflowll(number, 1073741824)) 5029 return 0; 5030 number *= 1073741824; 5031 5032 if(end[1] != '\0') 5033 /* trailing junk after multiplier, but 5034 * allow it to be "bytes" */ 5035 if(strcmp(end + 1, "bytes")) 5036 return 0; 5037 5038 break; 5039 case 'm': 5040 case 'M': 5041 if(multiply_overflowll(number, 1048576)) 5042 return 0; 5043 number *= 1048576; 5044 5045 if(end[1] != '\0') 5046 /* trailing junk after multiplier, but 5047 * allow it to be "bytes" */ 5048 if(strcmp(end + 1, "bytes")) 5049 return 0; 5050 5051 break; 5052 case 'k': 5053 case 'K': 5054 if(multiply_overflowll(number, 1024)) 5055 return 0; 5056 number *= 1024; 5057 5058 if(end[1] != '\0') 5059 /* trailing junk after multiplier, but 5060 * allow it to be "bytes" */ 5061 if(strcmp(end + 1, "bytes")) 5062 return 0; 5063 5064 break; 5065 case '\0': 5066 break; 5067 default: 5068 /* trailing junk after number */ 5069 return 0; 5070 } 5071 } else if(end[0] != '\0') 5072 /* trailing junk after number */ 5073 return 0; 5074 5075 *res = number; 5076 return 1; 5077 } 5078 5079 5080 int parse_number(char *start, int *res, int size) 5081 { 5082 long long number; 5083 5084 if(!parse_numberll(start, &number, size)) 5085 return 0; 5086 5087 /* check if long result will overflow signed int */ 5088 if(number > INT_MAX) 5089 return 0; 5090 5091 *res = (int) number; 5092 return 1; 5093 } 5094 5095 5096 int parse_num(char *arg, int *res) 5097 { 5098 return parse_number(arg, res, 0); 5099 } 5100 5101 5102 int get_physical_memory() 5103 { 5104 int phys_mem; 5105 #ifndef linux 5106 #ifdef HW_MEMSIZE 5107 #define SYSCTL_PHYSMEM HW_MEMSIZE 5108 #elif defined(HW_PHYSMEM64) 5109 #define SYSCTL_PHYSMEM HW_PHYSMEM64 5110 #else 5111 #define SYSCTL_PHYSMEM HW_PHYSMEM 5112 #endif 5113 5114 int mib[2]; 5115 uint64_t sysctl_physmem = 0; 5116 size_t sysctl_len = sizeof(sysctl_physmem); 5117 5118 mib[0] = CTL_HW; 5119 mib[1] = SYSCTL_PHYSMEM; 5120 5121 if(sysctl(mib, 2, &sysctl_physmem, &sysctl_len, NULL, 0) == 0) { 5122 /* some systems use 32-bit values, work with what we're given */ 5123 if (sysctl_len == 4) 5124 sysctl_physmem = *(uint32_t*)&sysctl_physmem; 5125 phys_mem = sysctl_physmem >> 20; 5126 } else { 5127 ERROR_START("Failed to get amount of available " 5128 "memory."); 5129 ERROR_EXIT(" Defaulting to least viable amount\n"); 5130 phys_mem = SQUASHFS_LOWMEM; 5131 } 5132 #undef SYSCTL_PHYSMEM 5133 #else 5134 /* Long longs are used here because with PAE, a 32-bit 5135 machine can have more than 4GB of physical memory */ 5136 5137 long long num_pages = sysconf(_SC_PHYS_PAGES); 5138 long long page_size = sysconf(_SC_PAGESIZE); 5139 phys_mem = num_pages * page_size >> 20; 5140 if(num_pages == -1 || page_size == -1) 5141 return 0; 5142 5143 #endif 5144 5145 if(phys_mem < SQUASHFS_LOWMEM) 5146 BAD_ERROR("Mksquashfs requires more physical memory than is " 5147 "available!\n"); 5148 5149 return phys_mem; 5150 } 5151 5152 5153 void check_usable_phys_mem(int total_mem) 5154 { 5155 /* 5156 * We want to allow users to use as much of their physical 5157 * memory as they wish. However, for practical reasons there are 5158 * limits which need to be imposed, to protect users from themselves 5159 * and to prevent people from using Mksquashfs as a DOS attack by using 5160 * all physical memory. Mksquashfs uses memory to cache data from disk 5161 * to optimise performance. It is pointless to ask it to use more 5162 * than 75% of physical memory, as this causes thrashing and it is thus 5163 * self-defeating. 5164 */ 5165 int mem = get_physical_memory(); 5166 5167 mem = (mem >> 1) + (mem >> 2); /* 75% */ 5168 5169 if(total_mem > mem && mem) { 5170 ERROR("Total memory requested is more than 75%% of physical " 5171 "memory.\n"); 5172 ERROR("Mksquashfs uses memory to cache data from disk to " 5173 "optimise performance.\n"); 5174 ERROR("It is pointless to ask it to use more than this amount " 5175 "of memory, as this\n"); 5176 ERROR("causes thrashing and it is thus self-defeating.\n"); 5177 BAD_ERROR("Requested memory size too large\n"); 5178 } 5179 5180 if(sizeof(void *) == 4 && total_mem > 2048) { 5181 /* 5182 * If we're running on a kernel with PAE or on a 64-bit kernel, 5183 * then the 75% physical memory limit can still easily exceed 5184 * the addressable memory by this process. 5185 * 5186 * Due to the typical kernel/user-space split (1GB/3GB, or 5187 * 2GB/2GB), we have to conservatively assume the 32-bit 5188 * processes can only address 2-3GB. So refuse if the user 5189 * tries to allocate more than 2GB. 5190 */ 5191 ERROR("Total memory requested may exceed maximum " 5192 "addressable memory by this process\n"); 5193 BAD_ERROR("Requested memory size too large\n"); 5194 } 5195 } 5196 5197 5198 int get_default_phys_mem() 5199 { 5200 /* 5201 * get_physical_memory() relies on /proc being mounted. 5202 * If it fails, issue a warning, and use 5203 * SQUASHFS_LOWMEM / SQUASHFS_TAKE as default, 5204 * and allow a larger value to be set with -mem. 5205 */ 5206 int mem = get_physical_memory(); 5207 5208 if(mem == 0) { 5209 mem = SQUASHFS_LOWMEM / SQUASHFS_TAKE; 5210 5211 ERROR("Warning: Cannot get size of physical memory, probably " 5212 "because /proc is missing.\n"); 5213 ERROR("Warning: Defaulting to minimal use of %d Mbytes, use " 5214 "-mem to set a better value,\n", mem); 5215 ERROR("Warning: or fix /proc.\n"); 5216 } else 5217 mem /= SQUASHFS_TAKE; 5218 5219 if(sizeof(void *) == 4 && mem > 640) { 5220 /* 5221 * If we're running on a kernel with PAE or on a 64-bit kernel, 5222 * the default memory usage can exceed the addressable 5223 * memory by this process. 5224 * Due to the typical kernel/user-space split (1GB/3GB, or 5225 * 2GB/2GB), we have to conservatively assume the 32-bit 5226 * processes can only address 2-3GB. So limit the default 5227 * usage to 640M, which gives room for other data. 5228 */ 5229 mem = 640; 5230 } 5231 5232 return mem; 5233 } 5234 5235 5236 void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq, 5237 int *fwriteq) 5238 { 5239 *readq = mem / SQUASHFS_READQ_MEM; 5240 *bwriteq = mem / SQUASHFS_BWRITEQ_MEM; 5241 *fwriteq = mem / SQUASHFS_FWRITEQ_MEM; 5242 *fragq = mem - *readq - *bwriteq - *fwriteq; 5243 } 5244 5245 5246 #define VERSION() \ 5247 printf("mksquashfs version 4.3-git (2014/09/12)\n");\ 5248 printf("copyright (C) 2014 Phillip Lougher "\ 5249 "<phillip (at) squashfs.org.uk>\n\n"); \ 5250 printf("This program is free software; you can redistribute it and/or"\ 5251 "\n");\ 5252 printf("modify it under the terms of the GNU General Public License"\ 5253 "\n");\ 5254 printf("as published by the Free Software Foundation; either version "\ 5255 "2,\n");\ 5256 printf("or (at your option) any later version.\n\n");\ 5257 printf("This program is distributed in the hope that it will be "\ 5258 "useful,\n");\ 5259 printf("but WITHOUT ANY WARRANTY; without even the implied warranty "\ 5260 "of\n");\ 5261 printf("MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the"\ 5262 "\n");\ 5263 printf("GNU General Public License for more details.\n"); 5264 int main(int argc, char *argv[]) 5265 { 5266 struct stat buf, source_buf; 5267 int res, i; 5268 char *b, *root_name = NULL; 5269 int keep_as_directory = FALSE; 5270 squashfs_inode inode; 5271 int readq; 5272 int fragq; 5273 int bwriteq; 5274 int fwriteq; 5275 int total_mem = get_default_phys_mem(); 5276 int progress = TRUE; 5277 int force_progress = FALSE; 5278 struct file_buffer **fragment = NULL; 5279 /* ANDROID CHANGES START*/ 5280 #ifdef ANDROID 5281 const char *fs_config_file = NULL; 5282 #endif 5283 /* ANDROID CHANGES END */ 5284 5285 if(argc > 1 && strcmp(argv[1], "-version") == 0) { 5286 VERSION(); 5287 exit(0); 5288 } 5289 5290 block_log = slog(block_size); 5291 calculate_queue_sizes(total_mem, &readq, &fragq, &bwriteq, &fwriteq); 5292 5293 for(i = 1; i < argc && argv[i][0] != '-'; i++); 5294 if(i < 3) 5295 goto printOptions; 5296 source_path = argv + 1; 5297 source = i - 2; 5298 5299 /* 5300 * Scan the command line for -comp xxx option, this is to ensure 5301 * any -X compressor specific options are passed to the 5302 * correct compressor 5303 */ 5304 for(; i < argc; i++) { 5305 struct compressor *prev_comp = comp; 5306 5307 if(strcmp(argv[i], "-comp") == 0) { 5308 if(++i == argc) { 5309 ERROR("%s: -comp missing compression type\n", 5310 argv[0]); 5311 exit(1); 5312 } 5313 comp = lookup_compressor(argv[i]); 5314 if(!comp->supported) { 5315 ERROR("%s: Compressor \"%s\" is not supported!" 5316 "\n", argv[0], argv[i]); 5317 ERROR("%s: Compressors available:\n", argv[0]); 5318 display_compressors("", COMP_DEFAULT); 5319 exit(1); 5320 } 5321 if(prev_comp != NULL && prev_comp != comp) { 5322 ERROR("%s: -comp multiple conflicting -comp" 5323 " options specified on command line" 5324 ", previously %s, now %s\n", argv[0], 5325 prev_comp->name, comp->name); 5326 exit(1); 5327 } 5328 compressor_opt_parsed = 1; 5329 5330 } else if(strcmp(argv[i], "-e") == 0) 5331 break; 5332 else if(strcmp(argv[i], "-root-becomes") == 0 || 5333 strcmp(argv[i], "-ef") == 0 || 5334 strcmp(argv[i], "-pf") == 0 || 5335 strcmp(argv[i], "-vaf") == 0 || 5336 strcmp(argv[i], "-comp") == 0) 5337 i++; 5338 } 5339 5340 /* 5341 * if no -comp option specified lookup default compressor. Note the 5342 * Makefile ensures the default compressor has been built, and so we 5343 * don't need to to check for failure here 5344 */ 5345 if(comp == NULL) 5346 comp = lookup_compressor(COMP_DEFAULT); 5347 5348 for(i = source + 2; i < argc; i++) { 5349 if(strcmp(argv[i], "-action") == 0 || 5350 strcmp(argv[i], "-a") ==0) { 5351 if(++i == argc) { 5352 ERROR("%s: %s missing action\n", 5353 argv[0], argv[i - 1]); 5354 exit(1); 5355 } 5356 res = parse_action(argv[i], ACTION_LOG_NONE); 5357 if(res == 0) 5358 exit(1); 5359 5360 } else if(strcmp(argv[i], "-verbose-action") == 0 || 5361 strcmp(argv[i], "-va") ==0) { 5362 if(++i == argc) { 5363 ERROR("%s: %s missing action\n", 5364 argv[0], argv[i - 1]); 5365 exit(1); 5366 } 5367 res = parse_action(argv[i], ACTION_LOG_VERBOSE); 5368 if(res == 0) 5369 exit(1); 5370 5371 } else if(strcmp(argv[i], "-true-action") == 0 || 5372 strcmp(argv[i], "-ta") ==0) { 5373 if(++i == argc) { 5374 ERROR("%s: %s missing action\n", 5375 argv[0], argv[i - 1]); 5376 exit(1); 5377 } 5378 res = parse_action(argv[i], ACTION_LOG_TRUE); 5379 if(res == 0) 5380 exit(1); 5381 5382 } else if(strcmp(argv[i], "-false-action") == 0 || 5383 strcmp(argv[i], "-fa") ==0) { 5384 if(++i == argc) { 5385 ERROR("%s: %s missing action\n", 5386 argv[0], argv[i - 1]); 5387 exit(1); 5388 } 5389 res = parse_action(argv[i], ACTION_LOG_FALSE); 5390 if(res == 0) 5391 exit(1); 5392 5393 } else if(strcmp(argv[i], "-action-file") == 0 || 5394 strcmp(argv[i], "-af") ==0) { 5395 if(++i == argc) { 5396 ERROR("%s: %s missing filename\n", argv[0], 5397 argv[i - 1]); 5398 exit(1); 5399 } 5400 if(read_action_file(argv[i], ACTION_LOG_NONE) == FALSE) 5401 exit(1); 5402 5403 } else if(strcmp(argv[i], "-verbose-action-file") == 0 || 5404 strcmp(argv[i], "-vaf") ==0) { 5405 if(++i == argc) { 5406 ERROR("%s: %s missing filename\n", argv[0], 5407 argv[i - 1]); 5408 exit(1); 5409 } 5410 if(read_action_file(argv[i], ACTION_LOG_VERBOSE) == FALSE) 5411 exit(1); 5412 5413 } else if(strcmp(argv[i], "-true-action-file") == 0 || 5414 strcmp(argv[i], "-taf") ==0) { 5415 if(++i == argc) { 5416 ERROR("%s: %s missing filename\n", argv[0], 5417 argv[i - 1]); 5418 exit(1); 5419 } 5420 if(read_action_file(argv[i], ACTION_LOG_TRUE) == FALSE) 5421 exit(1); 5422 5423 } else if(strcmp(argv[i], "-false-action-file") == 0 || 5424 strcmp(argv[i], "-faf") ==0) { 5425 if(++i == argc) { 5426 ERROR("%s: %s missing filename\n", argv[0], 5427 argv[i - 1]); 5428 exit(1); 5429 } 5430 if(read_action_file(argv[i], ACTION_LOG_FALSE) == FALSE) 5431 exit(1); 5432 5433 } else if(strcmp(argv[i], "-comp") == 0) 5434 /* parsed previously */ 5435 i++; 5436 5437 else if(strncmp(argv[i], "-X", 2) == 0) { 5438 int args; 5439 5440 if(strcmp(argv[i] + 2, "help") == 0) 5441 goto print_compressor_options; 5442 5443 args = compressor_options(comp, argv + i, argc - i); 5444 if(args < 0) { 5445 if(args == -1) { 5446 ERROR("%s: Unrecognised compressor" 5447 " option %s\n", argv[0], 5448 argv[i]); 5449 if(!compressor_opt_parsed) 5450 ERROR("%s: Did you forget to" 5451 " specify -comp?\n", 5452 argv[0]); 5453 print_compressor_options: 5454 ERROR("%s: selected compressor \"%s\"" 5455 ". Options supported: %s\n", 5456 argv[0], comp->name, 5457 comp->usage ? "" : "none"); 5458 if(comp->usage) 5459 comp->usage(); 5460 } 5461 exit(1); 5462 } 5463 i += args; 5464 5465 } else if(strcmp(argv[i], "-pf") == 0) { 5466 if(++i == argc) { 5467 ERROR("%s: -pf missing filename\n", argv[0]); 5468 exit(1); 5469 } 5470 if(read_pseudo_file(argv[i]) == FALSE) 5471 exit(1); 5472 } else if(strcmp(argv[i], "-p") == 0) { 5473 if(++i == argc) { 5474 ERROR("%s: -p missing pseudo file definition\n", 5475 argv[0]); 5476 exit(1); 5477 } 5478 if(read_pseudo_def(argv[i]) == FALSE) 5479 exit(1); 5480 } else if(strcmp(argv[i], "-recover") == 0) { 5481 if(++i == argc) { 5482 ERROR("%s: -recover missing recovery file\n", 5483 argv[0]); 5484 exit(1); 5485 } 5486 read_recovery_data(argv[i], argv[source + 1]); 5487 } else if(strcmp(argv[i], "-no-recovery") == 0) 5488 recover = FALSE; 5489 else if(strcmp(argv[i], "-wildcards") == 0) { 5490 old_exclude = FALSE; 5491 use_regex = FALSE; 5492 } else if(strcmp(argv[i], "-regex") == 0) { 5493 old_exclude = FALSE; 5494 use_regex = TRUE; 5495 } else if(strcmp(argv[i], "-no-sparse") == 0) 5496 sparse_files = FALSE; 5497 else if(strcmp(argv[i], "-no-progress") == 0) 5498 progress = FALSE; 5499 else if(strcmp(argv[i], "-progress") == 0) 5500 force_progress = TRUE; 5501 else if(strcmp(argv[i], "-no-exports") == 0) 5502 exportable = FALSE; 5503 else if(strcmp(argv[i], "-processors") == 0) { 5504 if((++i == argc) || !parse_num(argv[i], &processors)) { 5505 ERROR("%s: -processors missing or invalid " 5506 "processor number\n", argv[0]); 5507 exit(1); 5508 } 5509 if(processors < 1) { 5510 ERROR("%s: -processors should be 1 or larger\n", 5511 argv[0]); 5512 exit(1); 5513 } 5514 } else if(strcmp(argv[i], "-read-queue") == 0) { 5515 if((++i == argc) || !parse_num(argv[i], &readq)) { 5516 ERROR("%s: -read-queue missing or invalid " 5517 "queue size\n", argv[0]); 5518 exit(1); 5519 } 5520 if(readq < 1) { 5521 ERROR("%s: -read-queue should be 1 megabyte or " 5522 "larger\n", argv[0]); 5523 exit(1); 5524 } 5525 } else if(strcmp(argv[i], "-write-queue") == 0) { 5526 if((++i == argc) || !parse_num(argv[i], &bwriteq)) { 5527 ERROR("%s: -write-queue missing or invalid " 5528 "queue size\n", argv[0]); 5529 exit(1); 5530 } 5531 if(bwriteq < 2) { 5532 ERROR("%s: -write-queue should be 2 megabytes " 5533 "or larger\n", argv[0]); 5534 exit(1); 5535 } 5536 fwriteq = bwriteq >> 1; 5537 bwriteq -= fwriteq; 5538 } else if(strcmp(argv[i], "-fragment-queue") == 0) { 5539 if((++i == argc) || !parse_num(argv[i], &fragq)) { 5540 ERROR("%s: -fragment-queue missing or invalid " 5541 "queue size\n", argv[0]); 5542 exit(1); 5543 } 5544 if(fragq < 1) { 5545 ERROR("%s: -fragment-queue should be 1 " 5546 "megabyte or larger\n", argv[0]); 5547 exit(1); 5548 } 5549 } else if(strcmp(argv[i], "-mem") == 0) { 5550 long long number; 5551 5552 if((++i == argc) || 5553 !parse_numberll(argv[i], &number, 1)) { 5554 ERROR("%s: -mem missing or invalid mem size\n", 5555 argv[0]); 5556 exit(1); 5557 } 5558 5559 /* 5560 * convert from bytes to Mbytes, ensuring the value 5561 * does not overflow a signed int 5562 */ 5563 if(number >= (1LL << 51)) { 5564 ERROR("%s: -mem invalid mem size\n", argv[0]); 5565 exit(1); 5566 } 5567 5568 total_mem = number / 1048576; 5569 if(total_mem < (SQUASHFS_LOWMEM / SQUASHFS_TAKE)) { 5570 ERROR("%s: -mem should be %d Mbytes or " 5571 "larger\n", argv[0], 5572 SQUASHFS_LOWMEM / SQUASHFS_TAKE); 5573 exit(1); 5574 } 5575 calculate_queue_sizes(total_mem, &readq, &fragq, 5576 &bwriteq, &fwriteq); 5577 } else if(strcmp(argv[i], "-b") == 0) { 5578 if(++i == argc) { 5579 ERROR("%s: -b missing block size\n", argv[0]); 5580 exit(1); 5581 } 5582 if(!parse_number(argv[i], &block_size, 1)) { 5583 ERROR("%s: -b invalid block size\n", argv[0]); 5584 exit(1); 5585 } 5586 if((block_log = slog(block_size)) == 0) { 5587 ERROR("%s: -b block size not power of two or " 5588 "not between 4096 and 1Mbyte\n", 5589 argv[0]); 5590 exit(1); 5591 } 5592 } else if(strcmp(argv[i], "-ef") == 0) { 5593 if(++i == argc) { 5594 ERROR("%s: -ef missing filename\n", argv[0]); 5595 exit(1); 5596 } 5597 } else if(strcmp(argv[i], "-no-duplicates") == 0) 5598 duplicate_checking = FALSE; 5599 5600 else if(strcmp(argv[i], "-no-fragments") == 0) 5601 no_fragments = TRUE; 5602 5603 else if(strcmp(argv[i], "-always-use-fragments") == 0) 5604 always_use_fragments = TRUE; 5605 5606 else if(strcmp(argv[i], "-sort") == 0) { 5607 if(++i == argc) { 5608 ERROR("%s: -sort missing filename\n", argv[0]); 5609 exit(1); 5610 } 5611 } else if(strcmp(argv[i], "-all-root") == 0 || 5612 strcmp(argv[i], "-root-owned") == 0) 5613 global_uid = global_gid = 0; 5614 5615 else if(strcmp(argv[i], "-force-uid") == 0) { 5616 if(++i == argc) { 5617 ERROR("%s: -force-uid missing uid or user\n", 5618 argv[0]); 5619 exit(1); 5620 } 5621 if((global_uid = strtoll(argv[i], &b, 10)), *b =='\0') { 5622 if(global_uid < 0 || global_uid > 5623 (((long long) 1 << 32) - 1)) { 5624 ERROR("%s: -force-uid uid out of range" 5625 "\n", argv[0]); 5626 exit(1); 5627 } 5628 } else { 5629 struct passwd *uid = getpwnam(argv[i]); 5630 if(uid) 5631 global_uid = uid->pw_uid; 5632 else { 5633 ERROR("%s: -force-uid invalid uid or " 5634 "unknown user\n", argv[0]); 5635 exit(1); 5636 } 5637 } 5638 } else if(strcmp(argv[i], "-force-gid") == 0) { 5639 if(++i == argc) { 5640 ERROR("%s: -force-gid missing gid or group\n", 5641 argv[0]); 5642 exit(1); 5643 } 5644 if((global_gid = strtoll(argv[i], &b, 10)), *b =='\0') { 5645 if(global_gid < 0 || global_gid > 5646 (((long long) 1 << 32) - 1)) { 5647 ERROR("%s: -force-gid gid out of range" 5648 "\n", argv[0]); 5649 exit(1); 5650 } 5651 } else { 5652 struct group *gid = getgrnam(argv[i]); 5653 if(gid) 5654 global_gid = gid->gr_gid; 5655 else { 5656 ERROR("%s: -force-gid invalid gid or " 5657 "unknown group\n", argv[0]); 5658 exit(1); 5659 } 5660 } 5661 } else if(strcmp(argv[i], "-noI") == 0 || 5662 strcmp(argv[i], "-noInodeCompression") == 0) 5663 noI = TRUE; 5664 5665 else if(strcmp(argv[i], "-noD") == 0 || 5666 strcmp(argv[i], "-noDataCompression") == 0) 5667 noD = TRUE; 5668 5669 else if(strcmp(argv[i], "-noF") == 0 || 5670 strcmp(argv[i], "-noFragmentCompression") == 0) 5671 noF = TRUE; 5672 5673 else if(strcmp(argv[i], "-noX") == 0 || 5674 strcmp(argv[i], "-noXattrCompression") == 0) 5675 noX = TRUE; 5676 5677 else if(strcmp(argv[i], "-no-xattrs") == 0) 5678 no_xattrs = TRUE; 5679 5680 else if(strcmp(argv[i], "-xattrs") == 0) 5681 no_xattrs = FALSE; 5682 5683 /* ANDROID CHANGES START*/ 5684 #ifdef ANDROID 5685 else if(strcmp(argv[i], "-context-file") == 0) { 5686 if(++i == argc) { 5687 ERROR("%s: -context-file: missing file name\n", 5688 argv[0]); 5689 exit(1); 5690 } 5691 context_file = argv[i]; 5692 } 5693 else if(strcmp(argv[i], "-fs-config-file") == 0) { 5694 if(++i == argc) { 5695 ERROR("%s: -fs-config-file: missing file name\n", 5696 argv[0]); 5697 exit(1); 5698 } 5699 fs_config_file = argv[i]; 5700 } 5701 else if(strcmp(argv[i], "-t") == 0) { 5702 if(++i == argc) { 5703 ERROR("%s: -t missing compression threshold percentage\n", argv[0]); 5704 exit(1); 5705 } 5706 if(!parse_number(argv[i], &compress_thresh_per, 1)) { 5707 ERROR("%s: -t invalid compression threshold percentage\n", argv[0]); 5708 exit(1); 5709 } 5710 if(compress_thresh_per > 100 || compress_thresh_per < 0) { 5711 ERROR("%s: -t compression threshold percentage not between 0 and 100\n", 5712 argv[0]); 5713 exit(1); 5714 } 5715 } 5716 #endif 5717 /* ANDROID CHANGES END */ 5718 else if(strcmp(argv[i], "-nopad") == 0) 5719 nopad = TRUE; 5720 5721 else if(strcmp(argv[i], "-info") == 0) 5722 silent = FALSE; 5723 5724 else if(strcmp(argv[i], "-e") == 0) 5725 break; 5726 5727 else if(strcmp(argv[i], "-noappend") == 0) 5728 delete = TRUE; 5729 5730 else if(strcmp(argv[i], "-keep-as-directory") == 0) 5731 keep_as_directory = TRUE; 5732 /* ANDROID CHANGES START*/ 5733 #ifdef ANDROID 5734 else if(strcmp(argv[i], "-android-fs-config") == 0) 5735 android_config = TRUE; 5736 else if(strcmp(argv[i], "-mount-point") == 0) { 5737 if(++i == argc) { 5738 ERROR("%s: -mount-point: missing mount point name\n", 5739 argv[0]); 5740 exit(1); 5741 } 5742 mount_point = argv[i]; 5743 } 5744 else if(strcmp(argv[i], "-product-out") == 0) { 5745 if(++i == argc) { 5746 ERROR("%s: -product-out: missing path name\n", 5747 argv[0]); 5748 exit(1); 5749 } 5750 target_out_path = argv[i]; 5751 } 5752 else if(strcmp(argv[i], "-disable-4k-align") == 0) 5753 align_4k_blocks = FALSE; 5754 else if(strcmp(argv[i], "-block-map") == 0) { 5755 if(++i == argc) { 5756 ERROR("%s: -block-map: missing path name\n", 5757 argv[0]); 5758 exit(1); 5759 } 5760 block_map_file = fopen(argv[i], "w"); 5761 if (block_map_file == NULL) { 5762 ERROR("%s: -block-map: failed to open %s\n", 5763 argv[0], argv[i]); 5764 exit(1); 5765 } 5766 if (!align_4k_blocks) { 5767 ERROR("WARNING: Using block maps with unaligned 4k blocks " 5768 "is not ideal as block map offsets are multiples of 4k, " 5769 "consider not passing -disable-4k-align\n"); 5770 } 5771 } 5772 #endif 5773 /* ANDROID CHANGES END */ 5774 5775 else if(strcmp(argv[i], "-exit-on-error") == 0) 5776 exit_on_error = TRUE; 5777 5778 else if(strcmp(argv[i], "-root-becomes") == 0) { 5779 if(++i == argc) { 5780 ERROR("%s: -root-becomes: missing name\n", 5781 argv[0]); 5782 exit(1); 5783 } 5784 root_name = argv[i]; 5785 } else if(strcmp(argv[i], "-version") == 0) { 5786 VERSION(); 5787 } else { 5788 ERROR("%s: invalid option\n\n", argv[0]); 5789 printOptions: 5790 ERROR("SYNTAX:%s source1 source2 ... dest [options] " 5791 "[-e list of exclude\ndirs/files]\n", argv[0]); 5792 ERROR("\nFilesystem build options:\n"); 5793 ERROR("-comp <comp>\t\tselect <comp> compression\n"); 5794 ERROR("\t\t\tCompressors available:\n"); 5795 display_compressors("\t\t\t", COMP_DEFAULT); 5796 ERROR("-b <block_size>\t\tset data block to " 5797 "<block_size>. Default 128 Kbytes\n"); 5798 ERROR("\t\t\tOptionally a suffix of K or M can be" 5799 " given to specify\n\t\t\tKbytes or Mbytes" 5800 " respectively\n"); 5801 ERROR("-no-exports\t\tdon't make the filesystem " 5802 "exportable via NFS\n"); 5803 ERROR("-no-sparse\t\tdon't detect sparse files\n"); 5804 ERROR("-no-xattrs\t\tdon't store extended attributes" 5805 NOXOPT_STR "\n"); 5806 ERROR("-xattrs\t\t\tstore extended attributes" XOPT_STR 5807 "\n"); 5808 /* ANDROID CHANGES START*/ 5809 #ifdef ANDROID 5810 ERROR("-context-file <file>\tApply selinux security " 5811 "xattrs from context-file instead\n\t\t\t" 5812 "of reading xattrs from file system\n"); 5813 ERROR("-fs-config-file <file>\tAndroid specific " 5814 "filesystem config file\n"); 5815 ERROR("-t <compress_thresh>\tset minimum " 5816 "acceptable compression ratio of a block to\n\t\t\t" 5817 "<compress_thresh_per> otherwise don't compress. " 5818 "Default 0%\n"); 5819 #endif 5820 /* ANDROID CHANGES END */ 5821 ERROR("-noI\t\t\tdo not compress inode table\n"); 5822 ERROR("-noD\t\t\tdo not compress data blocks\n"); 5823 ERROR("-noF\t\t\tdo not compress fragment blocks\n"); 5824 ERROR("-noX\t\t\tdo not compress extended " 5825 "attributes\n"); 5826 ERROR("-no-fragments\t\tdo not use fragments\n"); 5827 ERROR("-always-use-fragments\tuse fragment blocks for " 5828 "files larger than block size\n"); 5829 ERROR("-no-duplicates\t\tdo not perform duplicate " 5830 "checking\n"); 5831 ERROR("-all-root\t\tmake all files owned by root\n"); 5832 ERROR("-force-uid uid\t\tset all file uids to uid\n"); 5833 ERROR("-force-gid gid\t\tset all file gids to gid\n"); 5834 ERROR("-nopad\t\t\tdo not pad filesystem to a multiple " 5835 "of 4K\n"); 5836 ERROR("-keep-as-directory\tif one source directory is " 5837 "specified, create a root\n"); 5838 ERROR("\t\t\tdirectory containing that directory, " 5839 "rather than the\n"); 5840 ERROR("\t\t\tcontents of the directory\n"); 5841 /* ANDROID CHANGES START*/ 5842 #ifdef ANDROID 5843 ERROR("-android-fs-config\tuse android fs config " 5844 "for mode, uid, and gids of inodes\n"); 5845 ERROR("-mount-point <name>\tNeed to be provided when " 5846 "android-fs-config or context-file\n\t\t\tare " 5847 "enabled and source directory is not mount point\n"); 5848 ERROR("-product-out <path>\tPRODUCT_OUT directory to " 5849 "read device specific FS rules files from\n"); 5850 ERROR("-disable-4k-align \tDon't 4k align data blocks. Default is false\n"); 5851 ERROR("-block-map <path>\tGenerate a block map for non-fragment files\n"); 5852 #endif 5853 /* ANDROID CHANGES END */ 5854 ERROR("\nFilesystem filter options:\n"); 5855 ERROR("-p <pseudo-definition>\tAdd pseudo file " 5856 "definition\n"); 5857 ERROR("-pf <pseudo-file>\tAdd list of pseudo file " 5858 "definitions\n"); 5859 ERROR("-sort <sort_file>\tsort files according to " 5860 "priorities in <sort_file>. One\n"); 5861 ERROR("\t\t\tfile or dir with priority per line. " 5862 "Priority -32768 to\n"); 5863 ERROR("\t\t\t32767, default priority 0\n"); 5864 ERROR("-ef <exclude_file>\tlist of exclude dirs/files." 5865 " One per line\n"); 5866 ERROR("-wildcards\t\tAllow extended shell wildcards " 5867 "(globbing) to be used in\n\t\t\texclude " 5868 "dirs/files\n"); 5869 ERROR("-regex\t\t\tAllow POSIX regular expressions to " 5870 "be used in exclude\n\t\t\tdirs/files\n"); 5871 ERROR("\nFilesystem append options:\n"); 5872 ERROR("-noappend\t\tdo not append to existing " 5873 "filesystem\n"); 5874 ERROR("-root-becomes <name>\twhen appending source " 5875 "files/directories, make the\n"); 5876 ERROR("\t\t\toriginal root become a subdirectory in " 5877 "the new root\n"); 5878 ERROR("\t\t\tcalled <name>, rather than adding the new " 5879 "source items\n"); 5880 ERROR("\t\t\tto the original root\n"); 5881 ERROR("\nMksquashfs runtime options:\n"); 5882 ERROR("-version\t\tprint version, licence and " 5883 "copyright message\n"); 5884 ERROR("-exit-on-error\t\ttreat normally ignored errors " 5885 "as fatal\n"); 5886 ERROR("-recover <name>\t\trecover filesystem data " 5887 "using recovery file <name>\n"); 5888 ERROR("-no-recovery\t\tdon't generate a recovery " 5889 "file\n"); 5890 ERROR("-info\t\t\tprint files written to filesystem\n"); 5891 ERROR("-no-progress\t\tdon't display the progress " 5892 "bar\n"); 5893 ERROR("-progress\t\tdisplay progress bar when using " 5894 "the -info option\n"); 5895 ERROR("-processors <number>\tUse <number> processors." 5896 " By default will use number of\n"); 5897 ERROR("\t\t\tprocessors available\n"); 5898 ERROR("-mem <size>\t\tUse <size> physical memory. " 5899 "Currently set to %dM\n", total_mem); 5900 ERROR("\t\t\tOptionally a suffix of K, M or G can be" 5901 " given to specify\n\t\t\tKbytes, Mbytes or" 5902 " Gbytes respectively\n"); 5903 ERROR("\nMiscellaneous options:\n"); 5904 ERROR("-root-owned\t\talternative name for -all-root" 5905 "\n"); 5906 ERROR("-noInodeCompression\talternative name for -noI" 5907 "\n"); 5908 ERROR("-noDataCompression\talternative name for -noD" 5909 "\n"); 5910 ERROR("-noFragmentCompression\talternative name for " 5911 "-noF\n"); 5912 ERROR("-noXattrCompression\talternative name for " 5913 "-noX\n"); 5914 ERROR("\n-Xhelp\t\t\tprint compressor options for" 5915 " selected compressor\n"); 5916 ERROR("\nCompressors available and compressor specific " 5917 "options:\n"); 5918 display_compressor_usage(COMP_DEFAULT); 5919 exit(1); 5920 } 5921 } 5922 5923 /* ANDROID CHANGES START*/ 5924 #ifdef ANDROID 5925 if (fs_config_file) { 5926 if (load_canned_fs_config(fs_config_file) < 0) { 5927 fprintf(stderr, "failed to load %s\n", fs_config_file); 5928 exit(1); 5929 } 5930 fs_config_func = canned_fs_config; 5931 } else if (mount_point) { 5932 fs_config_func = fs_config; 5933 } 5934 #endif 5935 /* ANDROID CHANGES END */ 5936 5937 /* 5938 * Some compressors may need the options to be checked for validity 5939 * once all the options have been processed 5940 */ 5941 res = compressor_options_post(comp, block_size); 5942 if(res) 5943 EXIT_MKSQUASHFS(); 5944 5945 /* 5946 * If the -info option has been selected then disable the 5947 * progress bar unless it has been explicitly enabled with 5948 * the -progress option 5949 */ 5950 if(!silent) 5951 progress = force_progress; 5952 5953 #ifdef SQUASHFS_TRACE 5954 /* 5955 * Disable progress bar if full debug tracing is enabled. 5956 * The progress bar in this case just gets in the way of the 5957 * debug trace output 5958 */ 5959 progress = FALSE; 5960 #endif 5961 5962 for(i = 0; i < source; i++) 5963 if(lstat(source_path[i], &source_buf) == -1) { 5964 fprintf(stderr, "Cannot stat source directory \"%s\" " 5965 "because %s\n", source_path[i], 5966 strerror(errno)); 5967 EXIT_MKSQUASHFS(); 5968 } 5969 5970 destination_file = argv[source + 1]; 5971 if(stat(argv[source + 1], &buf) == -1) { 5972 if(errno == ENOENT) { /* Does not exist */ 5973 fd = open(argv[source + 1], O_CREAT | O_TRUNC | O_RDWR, 5974 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); 5975 if(fd == -1) { 5976 perror("Could not create destination file"); 5977 exit(1); 5978 } 5979 delete = TRUE; 5980 } else { 5981 perror("Could not stat destination file"); 5982 exit(1); 5983 } 5984 5985 } else { 5986 if(S_ISBLK(buf.st_mode)) { 5987 if((fd = open(argv[source + 1], O_RDWR)) == -1) { 5988 perror("Could not open block device as " 5989 "destination"); 5990 exit(1); 5991 } 5992 block_device = 1; 5993 5994 } else if(S_ISREG(buf.st_mode)) { 5995 fd = open(argv[source + 1], (delete ? O_TRUNC : 0) | 5996 O_RDWR); 5997 if(fd == -1) { 5998 perror("Could not open regular file for " 5999 "writing as destination"); 6000 exit(1); 6001 } 6002 } 6003 else { 6004 ERROR("Destination not block device or regular file\n"); 6005 exit(1); 6006 } 6007 6008 } 6009 6010 /* 6011 * process the exclude files - must be done afer destination file has 6012 * been possibly created 6013 */ 6014 for(i = source + 2; i < argc; i++) 6015 if(strcmp(argv[i], "-ef") == 0) 6016 /* 6017 * Note presence of filename arg has already 6018 * been checked 6019 */ 6020 process_exclude_file(argv[++i]); 6021 else if(strcmp(argv[i], "-e") == 0) 6022 break; 6023 else if(strcmp(argv[i], "-root-becomes") == 0 || 6024 strcmp(argv[i], "-sort") == 0 || 6025 strcmp(argv[i], "-pf") == 0 || 6026 strcmp(argv[i], "-af") == 0 || 6027 strcmp(argv[i], "-vaf") == 0 || 6028 strcmp(argv[i], "-comp") == 0) 6029 i++; 6030 6031 if(i != argc) { 6032 if(++i == argc) { 6033 ERROR("%s: -e missing arguments\n", argv[0]); 6034 EXIT_MKSQUASHFS(); 6035 } 6036 while(i < argc) 6037 if(old_exclude) 6038 old_add_exclude(argv[i++]); 6039 else 6040 add_exclude(argv[i++]); 6041 } 6042 6043 /* process the sort files - must be done afer the exclude files */ 6044 for(i = source + 2; i < argc; i++) 6045 if(strcmp(argv[i], "-sort") == 0) { 6046 int res = read_sort_file(argv[++i], source, 6047 source_path); 6048 if(res == FALSE) 6049 BAD_ERROR("Failed to read sort file\n"); 6050 sorted ++; 6051 } else if(strcmp(argv[i], "-e") == 0) 6052 break; 6053 else if(strcmp(argv[i], "-root-becomes") == 0 || 6054 strcmp(argv[i], "-ef") == 0 || 6055 strcmp(argv[i], "-pf") == 0 || 6056 strcmp(argv[i], "-af") == 0 || 6057 strcmp(argv[i], "-vaf") == 0 || 6058 strcmp(argv[i], "-comp") == 0) 6059 i++; 6060 6061 if(!delete) { 6062 comp = read_super(fd, &sBlk, argv[source + 1]); 6063 if(comp == NULL) { 6064 ERROR("Failed to read existing filesystem - will not " 6065 "overwrite - ABORTING!\n"); 6066 ERROR("To force Mksquashfs to write to this block " 6067 "device or file use -noappend\n"); 6068 EXIT_MKSQUASHFS(); 6069 } 6070 6071 block_log = slog(block_size = sBlk.block_size); 6072 noI = SQUASHFS_UNCOMPRESSED_INODES(sBlk.flags); 6073 noD = SQUASHFS_UNCOMPRESSED_DATA(sBlk.flags); 6074 noF = SQUASHFS_UNCOMPRESSED_FRAGMENTS(sBlk.flags); 6075 noX = SQUASHFS_UNCOMPRESSED_XATTRS(sBlk.flags); 6076 no_fragments = SQUASHFS_NO_FRAGMENTS(sBlk.flags); 6077 always_use_fragments = SQUASHFS_ALWAYS_FRAGMENTS(sBlk.flags); 6078 duplicate_checking = SQUASHFS_DUPLICATES(sBlk.flags); 6079 exportable = SQUASHFS_EXPORTABLE(sBlk.flags); 6080 no_xattrs = SQUASHFS_NO_XATTRS(sBlk.flags); 6081 comp_opts = SQUASHFS_COMP_OPTS(sBlk.flags); 6082 } 6083 6084 initialise_threads(readq, fragq, bwriteq, fwriteq, delete, 6085 destination_file); 6086 6087 res = compressor_init(comp, &stream, SQUASHFS_METADATA_SIZE, 0); 6088 if(res) 6089 BAD_ERROR("compressor_init failed\n"); 6090 6091 if(delete) { 6092 int size; 6093 void *comp_data = compressor_dump_options(comp, block_size, 6094 &size); 6095 6096 printf("Creating %d.%d filesystem on %s, block size %d.\n", 6097 SQUASHFS_MAJOR, SQUASHFS_MINOR, argv[source + 1], block_size); 6098 6099 /* 6100 * store any compressor specific options after the superblock, 6101 * and set the COMP_OPT flag to show that the filesystem has 6102 * compressor specfic options 6103 */ 6104 if(comp_data) { 6105 unsigned short c_byte = size | SQUASHFS_COMPRESSED_BIT; 6106 6107 SQUASHFS_INSWAP_SHORTS(&c_byte, 1); 6108 write_destination(fd, sizeof(struct squashfs_super_block), 6109 sizeof(c_byte), &c_byte); 6110 write_destination(fd, sizeof(struct squashfs_super_block) + 6111 sizeof(c_byte), size, comp_data); 6112 bytes = sizeof(struct squashfs_super_block) + sizeof(c_byte) 6113 + size; 6114 comp_opts = TRUE; 6115 } else 6116 bytes = sizeof(struct squashfs_super_block); 6117 } else { 6118 unsigned int last_directory_block, inode_dir_offset, 6119 inode_dir_file_size, root_inode_size, 6120 inode_dir_start_block, uncompressed_data, 6121 compressed_data, inode_dir_inode_number, 6122 inode_dir_parent_inode; 6123 unsigned int root_inode_start = 6124 SQUASHFS_INODE_BLK(sBlk.root_inode), 6125 root_inode_offset = 6126 SQUASHFS_INODE_OFFSET(sBlk.root_inode); 6127 6128 if((bytes = read_filesystem(root_name, fd, &sBlk, &inode_table, 6129 &data_cache, &directory_table, 6130 &directory_data_cache, &last_directory_block, 6131 &inode_dir_offset, &inode_dir_file_size, 6132 &root_inode_size, &inode_dir_start_block, 6133 &file_count, &sym_count, &dev_count, &dir_count, 6134 &fifo_count, &sock_count, &total_bytes, 6135 &total_inode_bytes, &total_directory_bytes, 6136 &inode_dir_inode_number, 6137 &inode_dir_parent_inode, add_old_root_entry, 6138 &fragment_table, &inode_lookup_table)) == 0) { 6139 ERROR("Failed to read existing filesystem - will not " 6140 "overwrite - ABORTING!\n"); 6141 ERROR("To force Mksquashfs to write to this block " 6142 "device or file use -noappend\n"); 6143 EXIT_MKSQUASHFS(); 6144 } 6145 if((append_fragments = fragments = sBlk.fragments)) { 6146 fragment_table = realloc((char *) fragment_table, 6147 ((fragments + FRAG_SIZE - 1) & ~(FRAG_SIZE - 1)) 6148 * sizeof(struct squashfs_fragment_entry)); 6149 if(fragment_table == NULL) 6150 BAD_ERROR("Out of memory in save filesystem state\n"); 6151 } 6152 6153 printf("Appending to existing %d.%d filesystem on %s, block " 6154 "size %d\n", SQUASHFS_MAJOR, SQUASHFS_MINOR, argv[source + 1], 6155 block_size); 6156 printf("All -b, -noI, -noD, -noF, -noX, no-duplicates, no-fragments, " 6157 "-always-use-fragments,\n-exportable and -comp options " 6158 "ignored\n"); 6159 printf("\nIf appending is not wanted, please re-run with " 6160 "-noappend specified!\n\n"); 6161 6162 compressed_data = (inode_dir_offset + inode_dir_file_size) & 6163 ~(SQUASHFS_METADATA_SIZE - 1); 6164 uncompressed_data = (inode_dir_offset + inode_dir_file_size) & 6165 (SQUASHFS_METADATA_SIZE - 1); 6166 6167 /* save original filesystem state for restoring ... */ 6168 sfragments = fragments; 6169 sbytes = bytes; 6170 sinode_count = sBlk.inodes; 6171 scache_bytes = root_inode_offset + root_inode_size; 6172 sdirectory_cache_bytes = uncompressed_data; 6173 sdata_cache = malloc(scache_bytes); 6174 if(sdata_cache == NULL) 6175 BAD_ERROR("Out of memory in save filesystem state\n"); 6176 sdirectory_data_cache = malloc(sdirectory_cache_bytes); 6177 if(sdirectory_data_cache == NULL) 6178 BAD_ERROR("Out of memory in save filesystem state\n"); 6179 memcpy(sdata_cache, data_cache, scache_bytes); 6180 memcpy(sdirectory_data_cache, directory_data_cache + 6181 compressed_data, sdirectory_cache_bytes); 6182 sinode_bytes = root_inode_start; 6183 stotal_bytes = total_bytes; 6184 stotal_inode_bytes = total_inode_bytes; 6185 stotal_directory_bytes = total_directory_bytes + 6186 compressed_data; 6187 sfile_count = file_count; 6188 ssym_count = sym_count; 6189 sdev_count = dev_count; 6190 sdir_count = dir_count + 1; 6191 sfifo_count = fifo_count; 6192 ssock_count = sock_count; 6193 sdup_files = dup_files; 6194 sid_count = id_count; 6195 write_recovery_data(&sBlk); 6196 save_xattrs(); 6197 appending = TRUE; 6198 6199 /* 6200 * set the filesystem state up to be able to append to the 6201 * original filesystem. The filesystem state differs depending 6202 * on whether we're appending to the original root directory, or 6203 * if the original root directory becomes a sub-directory 6204 * (root-becomes specified on command line, here root_name != 6205 * NULL) 6206 */ 6207 inode_bytes = inode_size = root_inode_start; 6208 directory_size = last_directory_block; 6209 cache_size = root_inode_offset + root_inode_size; 6210 directory_cache_size = inode_dir_offset + inode_dir_file_size; 6211 if(root_name) { 6212 sdirectory_bytes = last_directory_block; 6213 sdirectory_compressed_bytes = 0; 6214 root_inode_number = inode_dir_parent_inode; 6215 inode_no = sBlk.inodes + 2; 6216 directory_bytes = last_directory_block; 6217 directory_cache_bytes = uncompressed_data; 6218 memmove(directory_data_cache, directory_data_cache + 6219 compressed_data, uncompressed_data); 6220 cache_bytes = root_inode_offset + root_inode_size; 6221 add_old_root_entry(root_name, sBlk.root_inode, 6222 inode_dir_inode_number, SQUASHFS_DIR_TYPE); 6223 total_directory_bytes += compressed_data; 6224 dir_count ++; 6225 } else { 6226 sdirectory_compressed_bytes = last_directory_block - 6227 inode_dir_start_block; 6228 sdirectory_compressed = 6229 malloc(sdirectory_compressed_bytes); 6230 if(sdirectory_compressed == NULL) 6231 BAD_ERROR("Out of memory in save filesystem " 6232 "state\n"); 6233 memcpy(sdirectory_compressed, directory_table + 6234 inode_dir_start_block, 6235 sdirectory_compressed_bytes); 6236 sdirectory_bytes = inode_dir_start_block; 6237 root_inode_number = inode_dir_inode_number; 6238 inode_no = sBlk.inodes + 1; 6239 directory_bytes = inode_dir_start_block; 6240 directory_cache_bytes = inode_dir_offset; 6241 cache_bytes = root_inode_offset; 6242 } 6243 6244 inode_count = file_count + dir_count + sym_count + dev_count + 6245 fifo_count + sock_count; 6246 } 6247 6248 if(path) 6249 paths = add_subdir(paths, path); 6250 6251 dump_actions(); 6252 dump_pseudos(); 6253 6254 if(delete && !keep_as_directory && source == 1 && 6255 S_ISDIR(source_buf.st_mode)) 6256 dir_scan(&inode, source_path[0], scan1_readdir, progress); 6257 else if(!keep_as_directory && source == 1 && 6258 S_ISDIR(source_buf.st_mode)) 6259 dir_scan(&inode, source_path[0], scan1_single_readdir, progress); 6260 else 6261 dir_scan(&inode, "", scan1_encomp_readdir, progress); 6262 sBlk.root_inode = inode; 6263 sBlk.inodes = inode_count; 6264 sBlk.s_magic = SQUASHFS_MAGIC; 6265 sBlk.s_major = SQUASHFS_MAJOR; 6266 sBlk.s_minor = SQUASHFS_MINOR; 6267 sBlk.block_size = block_size; 6268 sBlk.block_log = block_log; 6269 sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments, 6270 always_use_fragments, duplicate_checking, exportable, 6271 no_xattrs, comp_opts); 6272 sBlk.mkfs_time = time(NULL); 6273 6274 disable_info(); 6275 6276 while((fragment = get_frag_action(fragment))) 6277 write_fragment(*fragment); 6278 unlock_fragments(); 6279 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 6280 pthread_mutex_lock(&fragment_mutex); 6281 while(fragments_outstanding) { 6282 pthread_mutex_unlock(&fragment_mutex); 6283 sched_yield(); 6284 pthread_mutex_lock(&fragment_mutex); 6285 } 6286 pthread_cleanup_pop(1); 6287 6288 queue_put(to_writer, NULL); 6289 if(queue_get(from_writer) != 0) 6290 EXIT_MKSQUASHFS(); 6291 6292 set_progressbar_state(FALSE); 6293 write_filesystem_tables(&sBlk, nopad); 6294 6295 /* ANDROID CHANGES START*/ 6296 #ifdef ANDROID 6297 if (block_map_file) 6298 fclose(block_map_file); 6299 #endif 6300 /* ANDROID CHANGES END */ 6301 6302 return 0; 6303 } 6304
