1 /* 2 * Create a squashfs filesystem. This is a highly compressed read only 3 * filesystem. 4 * 5 * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 6 * 2012, 2013, 2014 7 * Phillip Lougher <phillip (at) squashfs.org.uk> 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 2, 12 * or (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software 21 * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 22 * 23 * mksquashfs.c 24 */ 25 26 #define FALSE 0 27 #define TRUE 1 28 #define MAX_LINE 16384 29 30 #include <pwd.h> 31 #include <grp.h> 32 #include <time.h> 33 #include <unistd.h> 34 #include <stdio.h> 35 #include <stddef.h> 36 #include <sys/types.h> 37 #include <sys/stat.h> 38 #include <fcntl.h> 39 #include <errno.h> 40 #include <dirent.h> 41 #include <string.h> 42 #include <stdlib.h> 43 #include <signal.h> 44 #include <setjmp.h> 45 #include <sys/types.h> 46 #include <sys/mman.h> 47 #include <pthread.h> 48 #include <regex.h> 49 #include <fnmatch.h> 50 #include <sys/wait.h> 51 #include <limits.h> 52 #include <ctype.h> 53 54 #ifndef FNM_EXTMATCH /* glibc extension */ 55 #define FNM_EXTMATCH 0 56 #endif 57 58 #ifndef linux 59 #define __BYTE_ORDER BYTE_ORDER 60 #define __BIG_ENDIAN BIG_ENDIAN 61 #define __LITTLE_ENDIAN LITTLE_ENDIAN 62 #include <sys/sysctl.h> 63 #else 64 #include <endian.h> 65 #include <sys/sysinfo.h> 66 #endif 67 68 #include "squashfs_fs.h" 69 #include "squashfs_swap.h" 70 #include "mksquashfs.h" 71 #include "sort.h" 72 #include "pseudo.h" 73 #include "compressor.h" 74 #include "xattr.h" 75 #include "action.h" 76 #include "error.h" 77 #include "progressbar.h" 78 #include "info.h" 79 #include "caches-queues-lists.h" 80 #include "read_fs.h" 81 #include "restore.h" 82 #include "process_fragments.h" 83 84 /* ANDROID CHANGES START*/ 85 #ifdef ANDROID 86 #include "android.h" 87 #include "private/android_filesystem_config.h" 88 #include "private/canned_fs_config.h" 89 int android_config = FALSE; 90 char *context_file = NULL; 91 char *mount_point = NULL; 92 char *target_out_path = NULL; 93 fs_config_func_t fs_config_func = NULL; 94 int compress_thresh_per = 0; 95 int align_4k_blocks = TRUE; 96 FILE *block_map_file = NULL; 97 #endif 98 /* ANDROID CHANGES END */ 99 100 int delete = FALSE; 101 int fd; 102 struct squashfs_super_block sBlk; 103 104 /* filesystem flags for building */ 105 int comp_opts = FALSE; 106 int no_xattrs = XATTR_DEF; 107 int noX = FALSE; 108 int duplicate_checking = TRUE; 109 int noF = FALSE; 110 int no_fragments = FALSE; 111 int always_use_fragments = FALSE; 112 int noI = FALSE; 113 int noD = FALSE; 114 int silent = TRUE; 115 int exportable = TRUE; 116 int sparse_files = TRUE; 117 int old_exclude = TRUE; 118 int use_regex = FALSE; 119 int nopad = FALSE; 120 int exit_on_error = FALSE; 121 122 long long global_uid = -1, global_gid = -1; 123 124 /* superblock attributes */ 125 int block_size = SQUASHFS_FILE_SIZE, block_log; 126 unsigned int id_count = 0; 127 int file_count = 0, sym_count = 0, dev_count = 0, dir_count = 0, fifo_count = 0, 128 sock_count = 0; 129 130 /* ANDROID CHANGES START*/ 131 #ifdef ANDROID 132 int whitelisted_count = 0; 133 #endif 134 /* ANDROID CHANGES END */ 135 136 /* write position within data section */ 137 long long bytes = 0, total_bytes = 0; 138 139 /* in memory directory table - possibly compressed */ 140 char *directory_table = NULL; 141 unsigned int directory_bytes = 0, directory_size = 0, total_directory_bytes = 0; 142 143 /* cached directory table */ 144 char *directory_data_cache = NULL; 145 unsigned int directory_cache_bytes = 0, directory_cache_size = 0; 146 147 /* in memory inode table - possibly compressed */ 148 char *inode_table = NULL; 149 unsigned int inode_bytes = 0, inode_size = 0, total_inode_bytes = 0; 150 151 /* cached inode table */ 152 char *data_cache = NULL; 153 unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; 154 155 /* inode lookup table */ 156 squashfs_inode *inode_lookup_table = NULL; 157 158 /* in memory directory data */ 159 #define I_COUNT_SIZE 128 160 #define DIR_ENTRIES 32 161 #define INODE_HASH_SIZE 65536 162 #define INODE_HASH_MASK (INODE_HASH_SIZE - 1) 163 #define INODE_HASH(dev, ino) (ino & INODE_HASH_MASK) 164 165 struct cached_dir_index { 166 struct squashfs_dir_index index; 167 char *name; 168 }; 169 170 struct directory { 171 unsigned int start_block; 172 unsigned int size; 173 unsigned char *buff; 174 unsigned char *p; 175 unsigned int entry_count; 176 unsigned char *entry_count_p; 177 unsigned int i_count; 178 unsigned int i_size; 179 struct cached_dir_index *index; 180 unsigned char *index_count_p; 181 unsigned int inode_number; 182 }; 183 184 struct inode_info *inode_info[INODE_HASH_SIZE]; 185 186 /* hash tables used to do fast duplicate searches in duplicate check */ 187 struct file_info *dupl[65536]; 188 int dup_files = 0; 189 190 /* exclude file handling */ 191 /* list of exclude dirs/files */ 192 struct exclude_info { 193 dev_t st_dev; 194 ino_t st_ino; 195 }; 196 197 #define EXCLUDE_SIZE 8192 198 int exclude = 0; 199 struct exclude_info *exclude_paths = NULL; 200 int old_excluded(char *filename, struct stat *buf); 201 202 struct path_entry { 203 char *name; 204 regex_t *preg; 205 struct pathname *paths; 206 }; 207 208 struct pathname { 209 int names; 210 struct path_entry *name; 211 }; 212 213 struct pathnames { 214 int count; 215 struct pathname *path[0]; 216 }; 217 #define PATHS_ALLOC_SIZE 10 218 219 struct pathnames *paths = NULL; 220 struct pathname *path = NULL; 221 struct pathname *stickypath = NULL; 222 int excluded(char *name, struct pathnames *paths, struct pathnames **new); 223 224 int fragments = 0; 225 226 #define FRAG_SIZE 32768 227 228 struct squashfs_fragment_entry *fragment_table = NULL; 229 int fragments_outstanding = 0; 230 231 int fragments_locked = FALSE; 232 233 /* current inode number for directories and non directories */ 234 unsigned int inode_no = 1; 235 unsigned int root_inode_number = 0; 236 237 /* list of source dirs/files */ 238 int source = 0; 239 char **source_path; 240 241 /* list of root directory entries read from original filesystem */ 242 int old_root_entries = 0; 243 struct old_root_entry_info { 244 char *name; 245 struct inode_info inode; 246 }; 247 struct old_root_entry_info *old_root_entry; 248 249 /* restore orignal filesystem state if appending to existing filesystem is 250 * cancelled */ 251 int appending = FALSE; 252 char *sdata_cache, *sdirectory_data_cache, *sdirectory_compressed; 253 254 long long sbytes, stotal_bytes; 255 256 unsigned int sinode_bytes, scache_bytes, sdirectory_bytes, 257 sdirectory_cache_bytes, sdirectory_compressed_bytes, 258 stotal_inode_bytes, stotal_directory_bytes, 259 sinode_count = 0, sfile_count, ssym_count, sdev_count, 260 sdir_count, sfifo_count, ssock_count, sdup_files; 261 int sfragments; 262 int threads; 263 264 /* flag whether destination file is a block device */ 265 int block_device = FALSE; 266 267 /* flag indicating whether files are sorted using sort list(s) */ 268 int sorted = FALSE; 269 270 /* save destination file name for deleting on error */ 271 char *destination_file = NULL; 272 273 /* recovery file for abnormal exit on appending */ 274 char *recovery_file = NULL; 275 int recover = TRUE; 276 277 /* uid/gid mapping tables */ 278 #define UGID_ENTRIES 340 279 280 struct ugid_map_entry { 281 unsigned int child_id; 282 unsigned int parent_id; 283 unsigned int length; 284 }; 285 struct ugid_map_entry uid_mapping[UGID_ENTRIES], gid_mapping[UGID_ENTRIES]; 286 unsigned int uid_map_count = 0, gid_map_count = 0; 287 288 289 struct id *id_hash_table[ID_ENTRIES]; 290 struct id *id_table[SQUASHFS_IDS], *sid_table[SQUASHFS_IDS]; 291 unsigned int uid_count = 0, guid_count = 0; 292 unsigned int sid_count = 0, suid_count = 0, sguid_count = 0; 293 294 struct cache *reader_buffer, *fragment_buffer, *reserve_cache; 295 struct cache *bwriter_buffer, *fwriter_buffer; 296 struct queue *to_reader, *to_deflate, *to_writer, *from_writer, 297 *to_frag, *locked_fragment, *to_process_frag; 298 struct seq_queue *to_main; 299 pthread_t reader_thread, writer_thread, main_thread; 300 pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; 301 pthread_t *restore_thread = NULL; 302 pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER; 303 pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER; 304 pthread_mutex_t dup_mutex = PTHREAD_MUTEX_INITIALIZER; 305 306 /* user options that control parallelisation */ 307 int processors = -1; 308 int bwriter_size; 309 310 /* compression operations */ 311 struct compressor *comp = NULL; 312 int compressor_opt_parsed = FALSE; 313 void *stream = NULL; 314 315 /* xattr stats */ 316 unsigned int xattr_bytes = 0, total_xattr_bytes = 0; 317 318 /* fragment to file mapping used when appending */ 319 int append_fragments = 0; 320 struct append_file **file_mapping; 321 322 /* root of the in-core directory structure */ 323 struct dir_info *root_dir; 324 325 static char *read_from_disk(long long start, unsigned int avail_bytes); 326 void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, 327 int type); 328 struct file_info *duplicate(long long file_size, long long bytes, 329 unsigned int **block_list, long long *start, struct fragment **fragment, 330 struct file_buffer *file_buffer, int blocks, unsigned short checksum, 331 int checksum_flag); 332 struct dir_info *dir_scan1(char *, char *, struct pathnames *, 333 struct dir_ent *(_readdir)(struct dir_info *), int); 334 void dir_scan2(struct dir_info *dir, struct pseudo *pseudo); 335 void dir_scan3(struct dir_info *dir); 336 void dir_scan4(struct dir_info *dir); 337 void dir_scan5(struct dir_info *dir); 338 void dir_scan6(struct dir_info *dir); 339 void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info); 340 struct file_info *add_non_dup(long long file_size, long long bytes, 341 unsigned int *block_list, long long start, struct fragment *fragment, 342 unsigned short checksum, unsigned short fragment_checksum, 343 int checksum_flag, int checksum_frag_flag); 344 long long generic_write_table(int, void *, int, void *, int); 345 void restorefs(); 346 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth); 347 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad); 348 unsigned short get_checksum_mem(char *buff, int bytes); 349 void check_usable_phys_mem(int total_mem); 350 351 /* ANDROID CHANGES START*/ 352 #ifdef ANDROID 353 static int whitelisted(struct stat *buf); 354 static void add_whitelist_entry(char *filename, struct stat *buf); 355 static int add_whitelist(char *path); 356 static void process_whitelist_file(char *argv); 357 358 #define WHITELIST_SIZE 8192 359 int whitelist = 0; 360 361 struct whitelist_info { 362 dev_t st_dev; 363 ino_t st_ino; 364 }; 365 char *whitelist_filename = NULL; 366 struct whitelist_info *whitelist_paths = NULL; 367 #endif 368 /* ANDROID CHANGES END */ 369 370 void prep_exit() 371 { 372 if(restore_thread) { 373 if(pthread_self() == *restore_thread) { 374 /* 375 * Recursive failure when trying to restore filesystem! 376 * Nothing to do except to exit, otherwise we'll just 377 * appear to hang. The user should be able to restore 378 * from the recovery file (which is why it was added, in 379 * case of catastrophic failure in Mksquashfs) 380 */ 381 exit(1); 382 } else { 383 /* signal the restore thread to restore */ 384 pthread_kill(*restore_thread, SIGUSR1); 385 pthread_exit(NULL); 386 } 387 } else if(delete) { 388 if(destination_file && !block_device) 389 unlink(destination_file); 390 } else if(recovery_file) 391 unlink(recovery_file); 392 } 393 394 395 int add_overflow(int a, int b) 396 { 397 return (INT_MAX - a) < b; 398 } 399 400 401 int shift_overflow(int a, int shift) 402 { 403 return (INT_MAX >> shift) < a; 404 } 405 406 407 int multiply_overflow(int a, int multiplier) 408 { 409 return (INT_MAX / multiplier) < a; 410 } 411 412 413 int multiply_overflowll(long long a, int multiplier) 414 { 415 return (LLONG_MAX / multiplier) < a; 416 } 417 418 419 #define MKINODE(A) ((squashfs_inode)(((squashfs_inode) inode_bytes << 16) \ 420 + (((char *)A) - data_cache))) 421 422 423 void restorefs() 424 { 425 ERROR("Exiting - restoring original filesystem!\n\n"); 426 427 bytes = sbytes; 428 memcpy(data_cache, sdata_cache, cache_bytes = scache_bytes); 429 memcpy(directory_data_cache, sdirectory_data_cache, 430 sdirectory_cache_bytes); 431 directory_cache_bytes = sdirectory_cache_bytes; 432 inode_bytes = sinode_bytes; 433 directory_bytes = sdirectory_bytes; 434 memcpy(directory_table + directory_bytes, sdirectory_compressed, 435 sdirectory_compressed_bytes); 436 directory_bytes += sdirectory_compressed_bytes; 437 total_bytes = stotal_bytes; 438 total_inode_bytes = stotal_inode_bytes; 439 total_directory_bytes = stotal_directory_bytes; 440 inode_count = sinode_count; 441 file_count = sfile_count; 442 sym_count = ssym_count; 443 dev_count = sdev_count; 444 dir_count = sdir_count; 445 fifo_count = sfifo_count; 446 sock_count = ssock_count; 447 dup_files = sdup_files; 448 fragments = sfragments; 449 id_count = sid_count; 450 restore_xattrs(); 451 write_filesystem_tables(&sBlk, nopad); 452 exit(1); 453 } 454 455 456 void sighandler() 457 { 458 EXIT_MKSQUASHFS(); 459 } 460 461 462 int mangle2(void *strm, char *d, char *s, int size, 463 int block_size, int uncompressed, int data_block) 464 { 465 int error, c_byte = 0; 466 467 if(!uncompressed) { 468 c_byte = compressor_compress(comp, strm, d, s, size, block_size, 469 &error); 470 if(c_byte == -1) 471 BAD_ERROR("mangle2:: %s compress failed with error " 472 "code %d\n", comp->name, error); 473 } 474 475 if(c_byte == 0 || c_byte >= size || 476 (c_byte > (size * ((100.0 - compress_thresh_per) / 100.0)))) { 477 memcpy(d, s, size); 478 return size | (data_block ? SQUASHFS_COMPRESSED_BIT_BLOCK : 479 SQUASHFS_COMPRESSED_BIT); 480 } 481 482 return c_byte; 483 } 484 485 486 int mangle(char *d, char *s, int size, int block_size, 487 int uncompressed, int data_block) 488 { 489 return mangle2(stream, d, s, size, block_size, uncompressed, 490 data_block); 491 } 492 493 494 void *get_inode(int req_size) 495 { 496 int data_space; 497 unsigned short c_byte; 498 499 while(cache_bytes >= SQUASHFS_METADATA_SIZE) { 500 if((inode_size - inode_bytes) < 501 ((SQUASHFS_METADATA_SIZE << 1)) + 2) { 502 void *it = realloc(inode_table, inode_size + 503 (SQUASHFS_METADATA_SIZE << 1) + 2); 504 if(it == NULL) 505 MEM_ERROR(); 506 inode_table = it; 507 inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 508 } 509 510 c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, 511 data_cache, SQUASHFS_METADATA_SIZE, 512 SQUASHFS_METADATA_SIZE, noI, 0); 513 TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); 514 SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); 515 inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; 516 total_inode_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; 517 memmove(data_cache, data_cache + SQUASHFS_METADATA_SIZE, 518 cache_bytes - SQUASHFS_METADATA_SIZE); 519 cache_bytes -= SQUASHFS_METADATA_SIZE; 520 } 521 522 data_space = (cache_size - cache_bytes); 523 if(data_space < req_size) { 524 int realloc_size = cache_size == 0 ? 525 ((req_size + SQUASHFS_METADATA_SIZE) & 526 ~(SQUASHFS_METADATA_SIZE - 1)) : req_size - 527 data_space; 528 529 void *dc = realloc(data_cache, cache_size + 530 realloc_size); 531 if(dc == NULL) 532 MEM_ERROR(); 533 cache_size += realloc_size; 534 data_cache = dc; 535 } 536 537 cache_bytes += req_size; 538 539 return data_cache + cache_bytes - req_size; 540 } 541 542 543 int read_bytes(int fd, void *buff, int bytes) 544 { 545 int res, count; 546 547 for(count = 0; count < bytes; count += res) { 548 res = read(fd, buff + count, bytes - count); 549 if(res < 1) { 550 if(res == 0) 551 goto bytes_read; 552 else if(errno != EINTR) { 553 ERROR("Read failed because %s\n", 554 strerror(errno)); 555 return -1; 556 } else 557 res = 0; 558 } 559 } 560 561 bytes_read: 562 return count; 563 } 564 565 566 int read_fs_bytes(int fd, long long byte, int bytes, void *buff) 567 { 568 off_t off = byte; 569 int res = 1; 570 571 TRACE("read_fs_bytes: reading from position 0x%llx, bytes %d\n", 572 byte, bytes); 573 574 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 575 pthread_mutex_lock(&pos_mutex); 576 if(lseek(fd, off, SEEK_SET) == -1) { 577 ERROR("read_fs_bytes: Lseek on destination failed because %s, " 578 "offset=0x%llx\n", strerror(errno), off); 579 res = 0; 580 } else if(read_bytes(fd, buff, bytes) < bytes) { 581 ERROR("Read on destination failed\n"); 582 res = 0; 583 } 584 585 pthread_cleanup_pop(1); 586 return res; 587 } 588 589 590 int write_bytes(int fd, void *buff, int bytes) 591 { 592 int res, count; 593 594 for(count = 0; count < bytes; count += res) { 595 res = write(fd, buff + count, bytes - count); 596 if(res == -1) { 597 if(errno != EINTR) { 598 ERROR("Write failed because %s\n", 599 strerror(errno)); 600 return -1; 601 } 602 res = 0; 603 } 604 } 605 606 return 0; 607 } 608 609 610 void write_destination(int fd, long long byte, int bytes, void *buff) 611 { 612 off_t off = byte; 613 614 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 615 pthread_mutex_lock(&pos_mutex); 616 617 if(lseek(fd, off, SEEK_SET) == -1) { 618 ERROR("write_destination: Lseek on destination " 619 "failed because %s, offset=0x%llx\n", strerror(errno), 620 off); 621 BAD_ERROR("Probably out of space on output %s\n", 622 block_device ? "block device" : "filesystem"); 623 } 624 625 if(write_bytes(fd, buff, bytes) == -1) 626 BAD_ERROR("Failed to write to output %s\n", 627 block_device ? "block device" : "filesystem"); 628 629 pthread_cleanup_pop(1); 630 } 631 632 633 long long write_inodes() 634 { 635 unsigned short c_byte; 636 int avail_bytes; 637 char *datap = data_cache; 638 long long start_bytes = bytes; 639 640 while(cache_bytes) { 641 if(inode_size - inode_bytes < 642 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 643 void *it = realloc(inode_table, inode_size + 644 ((SQUASHFS_METADATA_SIZE << 1) + 2)); 645 if(it == NULL) 646 MEM_ERROR(); 647 inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 648 inode_table = it; 649 } 650 avail_bytes = cache_bytes > SQUASHFS_METADATA_SIZE ? 651 SQUASHFS_METADATA_SIZE : cache_bytes; 652 c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, datap, 653 avail_bytes, SQUASHFS_METADATA_SIZE, noI, 0); 654 TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); 655 SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); 656 inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; 657 total_inode_bytes += avail_bytes + BLOCK_OFFSET; 658 datap += avail_bytes; 659 cache_bytes -= avail_bytes; 660 } 661 662 write_destination(fd, bytes, inode_bytes, inode_table); 663 bytes += inode_bytes; 664 665 return start_bytes; 666 } 667 668 669 long long write_directories() 670 { 671 unsigned short c_byte; 672 int avail_bytes; 673 char *directoryp = directory_data_cache; 674 long long start_bytes = bytes; 675 676 while(directory_cache_bytes) { 677 if(directory_size - directory_bytes < 678 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 679 void *dt = realloc(directory_table, 680 directory_size + ((SQUASHFS_METADATA_SIZE << 1) 681 + 2)); 682 if(dt == NULL) 683 MEM_ERROR(); 684 directory_size += (SQUASHFS_METADATA_SIZE << 1) + 2; 685 directory_table = dt; 686 } 687 avail_bytes = directory_cache_bytes > SQUASHFS_METADATA_SIZE ? 688 SQUASHFS_METADATA_SIZE : directory_cache_bytes; 689 c_byte = mangle(directory_table + directory_bytes + 690 BLOCK_OFFSET, directoryp, avail_bytes, 691 SQUASHFS_METADATA_SIZE, noI, 0); 692 TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, 693 c_byte); 694 SQUASHFS_SWAP_SHORTS(&c_byte, 695 directory_table + directory_bytes, 1); 696 directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + 697 BLOCK_OFFSET; 698 total_directory_bytes += avail_bytes + BLOCK_OFFSET; 699 directoryp += avail_bytes; 700 directory_cache_bytes -= avail_bytes; 701 } 702 write_destination(fd, bytes, directory_bytes, directory_table); 703 bytes += directory_bytes; 704 705 return start_bytes; 706 } 707 708 709 long long write_id_table() 710 { 711 unsigned int id_bytes = SQUASHFS_ID_BYTES(id_count); 712 unsigned int p[id_count]; 713 int i; 714 715 TRACE("write_id_table: ids %d, id_bytes %d\n", id_count, id_bytes); 716 for(i = 0; i < id_count; i++) { 717 TRACE("write_id_table: id index %d, id %d", i, id_table[i]->id); 718 SQUASHFS_SWAP_INTS(&id_table[i]->id, p + i, 1); 719 } 720 721 return generic_write_table(id_bytes, p, 0, NULL, noI); 722 } 723 724 725 struct id *get_id(unsigned int id) 726 { 727 int hash = ID_HASH(id); 728 struct id *entry = id_hash_table[hash]; 729 730 for(; entry; entry = entry->next) 731 if(entry->id == id) 732 break; 733 734 return entry; 735 } 736 737 738 struct id *create_id(unsigned int id) 739 { 740 int hash = ID_HASH(id); 741 struct id *entry = malloc(sizeof(struct id)); 742 if(entry == NULL) 743 MEM_ERROR(); 744 entry->id = id; 745 entry->index = id_count ++; 746 entry->flags = 0; 747 entry->next = id_hash_table[hash]; 748 id_hash_table[hash] = entry; 749 id_table[entry->index] = entry; 750 return entry; 751 } 752 753 754 int resolve_child_ugid(unsigned int *ugid, 755 const struct ugid_map_entry *ugid_mapping, 756 unsigned int ugid_map_count) 757 { 758 unsigned int i; 759 760 for (i = 0; i < ugid_map_count; i++) { 761 if (ugid_mapping[i].parent_id <= *ugid && 762 *ugid < 763 ugid_mapping[i].parent_id + ugid_mapping[i].length) { 764 *ugid = ugid_mapping[i].child_id + *ugid - 765 ugid_mapping[i].parent_id; 766 return 1; 767 } 768 } 769 770 return 0; 771 } 772 773 774 unsigned int get_uid(unsigned int uid, int resolve) 775 { 776 struct id *entry; 777 778 if (resolve && !resolve_child_ugid(&uid, uid_mapping, uid_map_count)) 779 BAD_ERROR("uid not found in mapping: %d\n", uid); 780 entry = get_id(uid); 781 782 if(entry == NULL) { 783 if(id_count == SQUASHFS_IDS) 784 BAD_ERROR("Out of uids!\n"); 785 entry = create_id(uid); 786 } 787 788 if((entry->flags & ISA_UID) == 0) { 789 entry->flags |= ISA_UID; 790 uid_count ++; 791 } 792 793 return entry->index; 794 } 795 796 797 unsigned int get_guid(unsigned int guid, int resolve) 798 { 799 struct id *entry; 800 801 if (resolve && !resolve_child_ugid(&guid, gid_mapping, gid_map_count)) 802 BAD_ERROR("gid not found in mapping: %d\n", guid); 803 entry = get_id(guid); 804 805 if(entry == NULL) { 806 if(id_count == SQUASHFS_IDS) 807 BAD_ERROR("Out of gids!\n"); 808 entry = create_id(guid); 809 } 810 811 if((entry->flags & ISA_GID) == 0) { 812 entry->flags |= ISA_GID; 813 guid_count ++; 814 } 815 816 return entry->index; 817 } 818 819 820 #define ALLOC_SIZE 128 821 822 char *_pathname(struct dir_ent *dir_ent, char *pathname, int *size) 823 { 824 if(pathname == NULL) { 825 pathname = malloc(ALLOC_SIZE); 826 if(pathname == NULL) 827 MEM_ERROR(); 828 } 829 830 for(;;) { 831 int res = snprintf(pathname, *size, "%s/%s", 832 dir_ent->our_dir->pathname, 833 dir_ent->source_name ? : dir_ent->name); 834 835 if(res < 0) 836 BAD_ERROR("snprintf failed in pathname\n"); 837 else if(res >= *size) { 838 /* 839 * pathname is too small to contain the result, so 840 * increase it and try again 841 */ 842 *size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); 843 pathname = realloc(pathname, *size); 844 if(pathname == NULL) 845 MEM_ERROR(); 846 } else 847 break; 848 } 849 850 return pathname; 851 } 852 853 854 char *pathname(struct dir_ent *dir_ent) 855 { 856 static char *pathname = NULL; 857 static int size = ALLOC_SIZE; 858 859 if (dir_ent->nonstandard_pathname) 860 return dir_ent->nonstandard_pathname; 861 862 return pathname = _pathname(dir_ent, pathname, &size); 863 } 864 865 866 char *pathname_reader(struct dir_ent *dir_ent) 867 { 868 static char *pathname = NULL; 869 static int size = ALLOC_SIZE; 870 871 if (dir_ent->nonstandard_pathname) 872 return dir_ent->nonstandard_pathname; 873 874 return pathname = _pathname(dir_ent, pathname, &size); 875 } 876 877 878 char *subpathname(struct dir_ent *dir_ent) 879 { 880 static char *subpath = NULL; 881 static int size = ALLOC_SIZE; 882 int res; 883 884 if(subpath == NULL) { 885 subpath = malloc(ALLOC_SIZE); 886 if(subpath == NULL) 887 MEM_ERROR(); 888 } 889 890 for(;;) { 891 if(dir_ent->our_dir->subpath[0] != '\0') 892 res = snprintf(subpath, size, "%s/%s", 893 dir_ent->our_dir->subpath, dir_ent->name); 894 else 895 res = snprintf(subpath, size, "/%s", dir_ent->name); 896 897 if(res < 0) 898 BAD_ERROR("snprintf failed in subpathname\n"); 899 else if(res >= size) { 900 /* 901 * subpath is too small to contain the result, so 902 * increase it and try again 903 */ 904 size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); 905 subpath = realloc(subpath, size); 906 if(subpath == NULL) 907 MEM_ERROR(); 908 } else 909 break; 910 } 911 912 return subpath; 913 } 914 915 916 static inline unsigned int get_inode_no(struct inode_info *inode) 917 { 918 return inode->inode_number; 919 } 920 921 922 static inline unsigned int get_parent_no(struct dir_info *dir) 923 { 924 return dir->depth ? get_inode_no(dir->dir_ent->inode) : inode_no; 925 } 926 927 928 /* ANDROID CHANGES START*/ 929 #ifdef ANDROID 930 931 /* Round up the passed |n| value to the smallest multiple of 4096 greater or 932 * equal than |n| and return the 4K-block number for that value. */ 933 static unsigned long long round_up_block(unsigned long long n) { 934 const unsigned long long kMapBlockSize = 4096; 935 return (n + kMapBlockSize - 1) / kMapBlockSize; 936 } 937 938 static inline void write_block_map_entry(char *sub_path, unsigned long long start_block, unsigned long long total_size, 939 char * mount_point, FILE *block_map_file) { 940 if (block_map_file) { 941 /* We assign each 4K block based on what file the first byte of the block 942 * belongs to. The current file consists of the chunk of bytes in the 943 * interval [start_block, start_block + total_size), (closed on the left end 944 * and open on the right end). We then compute the first block whose first 945 * byte is equal to or greater than start_block as |round_start| and then 946 * the first block whose first byte is *past* this interval, as 947 * |round_end + 1|. This means that the blocks that should be assigned to 948 * the current file are in the interval [round_start, round_end + 1), or 949 * simply [round_start, round_end]. 950 */ 951 unsigned long long round_start = round_up_block(start_block); 952 unsigned long long round_end = round_up_block(start_block + total_size) - 1; 953 if (round_start && total_size && round_start <= round_end) { 954 fprintf(block_map_file, "/%s", mount_point); 955 if (sub_path[0] != '/') fprintf(block_map_file, "/"); 956 if (round_start == round_end) 957 fprintf(block_map_file, "%s %lld\n", sub_path, round_start); 958 else 959 fprintf(block_map_file, "%s %lld-%lld\n", sub_path, round_start, round_end); 960 } 961 } 962 } 963 #endif 964 /* ANDROID CHANGES END */ 965 966 int create_inode(squashfs_inode *i_no, struct dir_info *dir_info, 967 struct dir_ent *dir_ent, int type, long long byte_size, 968 long long start_block, unsigned int offset, unsigned int *block_list, 969 struct fragment *fragment, struct directory *dir_in, long long sparse) 970 { 971 struct stat *buf = &dir_ent->inode->buf; 972 union squashfs_inode_header inode_header; 973 struct squashfs_base_inode_header *base = &inode_header.base; 974 void *inode; 975 char *filename = pathname(dir_ent); 976 int nlink = dir_ent->inode->nlink; 977 int xattr = read_xattrs(dir_ent); 978 979 switch(type) { 980 case SQUASHFS_FILE_TYPE: 981 if(dir_ent->inode->nlink > 1 || 982 byte_size >= (1LL << 32) || 983 start_block >= (1LL << 32) || 984 sparse || IS_XATTR(xattr)) 985 type = SQUASHFS_LREG_TYPE; 986 break; 987 case SQUASHFS_DIR_TYPE: 988 if(dir_info->dir_is_ldir || IS_XATTR(xattr)) 989 type = SQUASHFS_LDIR_TYPE; 990 break; 991 case SQUASHFS_SYMLINK_TYPE: 992 if(IS_XATTR(xattr)) 993 type = SQUASHFS_LSYMLINK_TYPE; 994 break; 995 case SQUASHFS_BLKDEV_TYPE: 996 if(IS_XATTR(xattr)) 997 type = SQUASHFS_LBLKDEV_TYPE; 998 break; 999 case SQUASHFS_CHRDEV_TYPE: 1000 if(IS_XATTR(xattr)) 1001 type = SQUASHFS_LCHRDEV_TYPE; 1002 break; 1003 case SQUASHFS_FIFO_TYPE: 1004 if(IS_XATTR(xattr)) 1005 type = SQUASHFS_LFIFO_TYPE; 1006 break; 1007 case SQUASHFS_SOCKET_TYPE: 1008 if(IS_XATTR(xattr)) 1009 type = SQUASHFS_LSOCKET_TYPE; 1010 break; 1011 } 1012 1013 base->mode = SQUASHFS_MODE(buf->st_mode); 1014 base->uid = get_uid((unsigned int) global_uid == -1 ? 1015 buf->st_uid : global_uid, 1); 1016 base->inode_type = type; 1017 base->guid = get_guid((unsigned int) global_gid == -1 ? 1018 buf->st_gid : global_gid, 1); 1019 base->mtime = buf->st_mtime; 1020 base->inode_number = get_inode_no(dir_ent->inode); 1021 1022 if(type == SQUASHFS_FILE_TYPE) { 1023 int i; 1024 struct squashfs_reg_inode_header *reg = &inode_header.reg; 1025 size_t off = offsetof(struct squashfs_reg_inode_header, block_list); 1026 /* ANDROID CHANGES START*/ 1027 #ifdef ANDROID 1028 unsigned long long total_size = 0; 1029 char *sub_path; 1030 #endif 1031 /* ANDROID CHANGES END */ 1032 1033 inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); 1034 reg->file_size = byte_size; 1035 reg->start_block = start_block; 1036 reg->fragment = fragment->index; 1037 reg->offset = fragment->offset; 1038 SQUASHFS_SWAP_REG_INODE_HEADER(reg, inode); 1039 SQUASHFS_SWAP_INTS(block_list, inode + off, offset); 1040 TRACE("File inode, file_size %lld, start_block 0x%llx, blocks " 1041 "%d, fragment %d, offset %d, size %d\n", byte_size, 1042 start_block, offset, fragment->index, fragment->offset, 1043 fragment->size); 1044 for(i = 0; i < offset; i++) { 1045 TRACE("Block %d, size %d\n", i, block_list[i]); 1046 total_size += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]); 1047 } 1048 /* ANDROID CHANGES START*/ 1049 #ifdef ANDROID 1050 sub_path = subpathname(dir_ent); 1051 if (block_map_file && fragment->index == -1) { 1052 write_block_map_entry(sub_path, start_block, total_size, mount_point, block_map_file); 1053 } 1054 #endif 1055 /* ANDROID CHANGES END */ 1056 } 1057 else if(type == SQUASHFS_LREG_TYPE) { 1058 /* ANDROID CHANGES START*/ 1059 #ifdef ANDROID 1060 unsigned long long total_size = 0; 1061 char *sub_path; 1062 #endif 1063 /* ANDROID CHANGES END */ 1064 int i; 1065 struct squashfs_lreg_inode_header *reg = &inode_header.lreg; 1066 size_t off = offsetof(struct squashfs_lreg_inode_header, block_list); 1067 1068 inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); 1069 reg->nlink = nlink; 1070 reg->file_size = byte_size; 1071 reg->start_block = start_block; 1072 reg->fragment = fragment->index; 1073 reg->offset = fragment->offset; 1074 if(sparse && sparse >= byte_size) 1075 sparse = byte_size - 1; 1076 reg->sparse = sparse; 1077 reg->xattr = xattr; 1078 SQUASHFS_SWAP_LREG_INODE_HEADER(reg, inode); 1079 SQUASHFS_SWAP_INTS(block_list, inode + off, offset); 1080 TRACE("Long file inode, file_size %lld, start_block 0x%llx, " 1081 "blocks %d, fragment %d, offset %d, size %d, nlink %d" 1082 "\n", byte_size, start_block, offset, fragment->index, 1083 fragment->offset, fragment->size, nlink); 1084 for(i = 0; i < offset; i++) { 1085 TRACE("Block %d, size %d\n", i, block_list[i]); 1086 total_size += SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[i]); 1087 } 1088 /* ANDROID CHANGES START*/ 1089 #ifdef ANDROID 1090 sub_path = subpathname(dir_ent); 1091 if (block_map_file && fragment->index == -1) { 1092 write_block_map_entry(sub_path, start_block, total_size, mount_point, block_map_file); 1093 } 1094 #endif 1095 /* ANDROID CHANGES END */ 1096 } 1097 else if(type == SQUASHFS_LDIR_TYPE) { 1098 int i; 1099 unsigned char *p; 1100 struct squashfs_ldir_inode_header *dir = &inode_header.ldir; 1101 struct cached_dir_index *index = dir_in->index; 1102 unsigned int i_count = dir_in->i_count; 1103 unsigned int i_size = dir_in->i_size; 1104 1105 if(byte_size >= 1 << 27) 1106 BAD_ERROR("directory greater than 2^27-1 bytes!\n"); 1107 1108 inode = get_inode(sizeof(*dir) + i_size); 1109 dir->inode_type = SQUASHFS_LDIR_TYPE; 1110 dir->nlink = dir_ent->dir->directory_count + 2; 1111 dir->file_size = byte_size; 1112 dir->offset = offset; 1113 dir->start_block = start_block; 1114 dir->i_count = i_count; 1115 dir->parent_inode = get_parent_no(dir_ent->our_dir); 1116 dir->xattr = xattr; 1117 1118 SQUASHFS_SWAP_LDIR_INODE_HEADER(dir, inode); 1119 p = inode + offsetof(struct squashfs_ldir_inode_header, index); 1120 for(i = 0; i < i_count; i++) { 1121 SQUASHFS_SWAP_DIR_INDEX(&index[i].index, p); 1122 p += offsetof(struct squashfs_dir_index, name); 1123 memcpy(p, index[i].name, index[i].index.size + 1); 1124 p += index[i].index.size + 1; 1125 } 1126 TRACE("Long directory inode, file_size %lld, start_block " 1127 "0x%llx, offset 0x%x, nlink %d\n", byte_size, 1128 start_block, offset, dir_ent->dir->directory_count + 2); 1129 } 1130 else if(type == SQUASHFS_DIR_TYPE) { 1131 struct squashfs_dir_inode_header *dir = &inode_header.dir; 1132 1133 inode = get_inode(sizeof(*dir)); 1134 dir->nlink = dir_ent->dir->directory_count + 2; 1135 dir->file_size = byte_size; 1136 dir->offset = offset; 1137 dir->start_block = start_block; 1138 dir->parent_inode = get_parent_no(dir_ent->our_dir); 1139 SQUASHFS_SWAP_DIR_INODE_HEADER(dir, inode); 1140 TRACE("Directory inode, file_size %lld, start_block 0x%llx, " 1141 "offset 0x%x, nlink %d\n", byte_size, start_block, 1142 offset, dir_ent->dir->directory_count + 2); 1143 } 1144 else if(type == SQUASHFS_CHRDEV_TYPE || type == SQUASHFS_BLKDEV_TYPE) { 1145 struct squashfs_dev_inode_header *dev = &inode_header.dev; 1146 unsigned int major = major(buf->st_rdev); 1147 unsigned int minor = minor(buf->st_rdev); 1148 1149 if(major > 0xfff) { 1150 ERROR("Major %d out of range in device node %s, " 1151 "truncating to %d\n", major, filename, 1152 major & 0xfff); 1153 major &= 0xfff; 1154 } 1155 if(minor > 0xfffff) { 1156 ERROR("Minor %d out of range in device node %s, " 1157 "truncating to %d\n", minor, filename, 1158 minor & 0xfffff); 1159 minor &= 0xfffff; 1160 } 1161 inode = get_inode(sizeof(*dev)); 1162 dev->nlink = nlink; 1163 dev->rdev = (major << 8) | (minor & 0xff) | 1164 ((minor & ~0xff) << 12); 1165 SQUASHFS_SWAP_DEV_INODE_HEADER(dev, inode); 1166 TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); 1167 } 1168 else if(type == SQUASHFS_LCHRDEV_TYPE || type == SQUASHFS_LBLKDEV_TYPE) { 1169 struct squashfs_ldev_inode_header *dev = &inode_header.ldev; 1170 unsigned int major = major(buf->st_rdev); 1171 unsigned int minor = minor(buf->st_rdev); 1172 1173 if(major > 0xfff) { 1174 ERROR("Major %d out of range in device node %s, " 1175 "truncating to %d\n", major, filename, 1176 major & 0xfff); 1177 major &= 0xfff; 1178 } 1179 if(minor > 0xfffff) { 1180 ERROR("Minor %d out of range in device node %s, " 1181 "truncating to %d\n", minor, filename, 1182 minor & 0xfffff); 1183 minor &= 0xfffff; 1184 } 1185 inode = get_inode(sizeof(*dev)); 1186 dev->nlink = nlink; 1187 dev->rdev = (major << 8) | (minor & 0xff) | 1188 ((minor & ~0xff) << 12); 1189 dev->xattr = xattr; 1190 SQUASHFS_SWAP_LDEV_INODE_HEADER(dev, inode); 1191 TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); 1192 } 1193 else if(type == SQUASHFS_SYMLINK_TYPE) { 1194 struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; 1195 int byte = strlen(dir_ent->inode->symlink); 1196 size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); 1197 1198 inode = get_inode(sizeof(*symlink) + byte); 1199 symlink->nlink = nlink; 1200 symlink->symlink_size = byte; 1201 SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); 1202 strncpy(inode + off, dir_ent->inode->symlink, byte); 1203 TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, 1204 nlink); 1205 } 1206 else if(type == SQUASHFS_LSYMLINK_TYPE) { 1207 struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; 1208 int byte = strlen(dir_ent->inode->symlink); 1209 size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); 1210 1211 inode = get_inode(sizeof(*symlink) + byte + 1212 sizeof(unsigned int)); 1213 symlink->nlink = nlink; 1214 symlink->symlink_size = byte; 1215 SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); 1216 strncpy(inode + off, dir_ent->inode->symlink, byte); 1217 SQUASHFS_SWAP_INTS(&xattr, inode + off + byte, 1); 1218 TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, 1219 nlink); 1220 } 1221 else if(type == SQUASHFS_FIFO_TYPE || type == SQUASHFS_SOCKET_TYPE) { 1222 struct squashfs_ipc_inode_header *ipc = &inode_header.ipc; 1223 1224 inode = get_inode(sizeof(*ipc)); 1225 ipc->nlink = nlink; 1226 SQUASHFS_SWAP_IPC_INODE_HEADER(ipc, inode); 1227 TRACE("ipc inode, type %s, nlink %d\n", type == 1228 SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); 1229 } 1230 else if(type == SQUASHFS_LFIFO_TYPE || type == SQUASHFS_LSOCKET_TYPE) { 1231 struct squashfs_lipc_inode_header *ipc = &inode_header.lipc; 1232 1233 inode = get_inode(sizeof(*ipc)); 1234 ipc->nlink = nlink; 1235 ipc->xattr = xattr; 1236 SQUASHFS_SWAP_LIPC_INODE_HEADER(ipc, inode); 1237 TRACE("ipc inode, type %s, nlink %d\n", type == 1238 SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); 1239 } else 1240 BAD_ERROR("Unrecognised inode %d in create_inode\n", type); 1241 1242 *i_no = MKINODE(inode); 1243 inode_count ++; 1244 1245 TRACE("Created inode 0x%llx, type %d, uid %d, guid %d\n", *i_no, type, 1246 base->uid, base->guid); 1247 1248 return TRUE; 1249 } 1250 1251 1252 void add_dir(squashfs_inode inode, unsigned int inode_number, char *name, 1253 int type, struct directory *dir) 1254 { 1255 unsigned char *buff; 1256 struct squashfs_dir_entry idir; 1257 unsigned int start_block = inode >> 16; 1258 unsigned int offset = inode & 0xffff; 1259 unsigned int size = strlen(name); 1260 size_t name_off = offsetof(struct squashfs_dir_entry, name); 1261 1262 if(size > SQUASHFS_NAME_LEN) { 1263 size = SQUASHFS_NAME_LEN; 1264 ERROR("Filename is greater than %d characters, truncating! ..." 1265 "\n", SQUASHFS_NAME_LEN); 1266 } 1267 1268 if(dir->p + sizeof(struct squashfs_dir_entry) + size + 1269 sizeof(struct squashfs_dir_header) 1270 >= dir->buff + dir->size) { 1271 buff = realloc(dir->buff, dir->size += SQUASHFS_METADATA_SIZE); 1272 if(buff == NULL) 1273 MEM_ERROR(); 1274 1275 dir->p = (dir->p - dir->buff) + buff; 1276 if(dir->entry_count_p) 1277 dir->entry_count_p = (dir->entry_count_p - dir->buff + 1278 buff); 1279 dir->index_count_p = dir->index_count_p - dir->buff + buff; 1280 dir->buff = buff; 1281 } 1282 1283 if(dir->entry_count == 256 || start_block != dir->start_block || 1284 ((dir->entry_count_p != NULL) && 1285 ((dir->p + sizeof(struct squashfs_dir_entry) + size - 1286 dir->index_count_p) > SQUASHFS_METADATA_SIZE)) || 1287 ((long long) inode_number - dir->inode_number) > 32767 1288 || ((long long) inode_number - dir->inode_number) 1289 < -32768) { 1290 if(dir->entry_count_p) { 1291 struct squashfs_dir_header dir_header; 1292 1293 if((dir->p + sizeof(struct squashfs_dir_entry) + size - 1294 dir->index_count_p) > 1295 SQUASHFS_METADATA_SIZE) { 1296 if(dir->i_count % I_COUNT_SIZE == 0) { 1297 dir->index = realloc(dir->index, 1298 (dir->i_count + I_COUNT_SIZE) * 1299 sizeof(struct cached_dir_index)); 1300 if(dir->index == NULL) 1301 MEM_ERROR(); 1302 } 1303 dir->index[dir->i_count].index.index = 1304 dir->p - dir->buff; 1305 dir->index[dir->i_count].index.size = size - 1; 1306 dir->index[dir->i_count++].name = name; 1307 dir->i_size += sizeof(struct squashfs_dir_index) 1308 + size; 1309 dir->index_count_p = dir->p; 1310 } 1311 1312 dir_header.count = dir->entry_count - 1; 1313 dir_header.start_block = dir->start_block; 1314 dir_header.inode_number = dir->inode_number; 1315 SQUASHFS_SWAP_DIR_HEADER(&dir_header, 1316 dir->entry_count_p); 1317 1318 } 1319 1320 1321 dir->entry_count_p = dir->p; 1322 dir->start_block = start_block; 1323 dir->entry_count = 0; 1324 dir->inode_number = inode_number; 1325 dir->p += sizeof(struct squashfs_dir_header); 1326 } 1327 1328 idir.offset = offset; 1329 idir.type = type; 1330 idir.size = size - 1; 1331 idir.inode_number = ((long long) inode_number - dir->inode_number); 1332 SQUASHFS_SWAP_DIR_ENTRY(&idir, dir->p); 1333 strncpy((char *) dir->p + name_off, name, size); 1334 dir->p += sizeof(struct squashfs_dir_entry) + size; 1335 dir->entry_count ++; 1336 } 1337 1338 1339 void write_dir(squashfs_inode *inode, struct dir_info *dir_info, 1340 struct directory *dir) 1341 { 1342 unsigned int dir_size = dir->p - dir->buff; 1343 int data_space = directory_cache_size - directory_cache_bytes; 1344 unsigned int directory_block, directory_offset, i_count, index; 1345 unsigned short c_byte; 1346 1347 if(data_space < dir_size) { 1348 int realloc_size = directory_cache_size == 0 ? 1349 ((dir_size + SQUASHFS_METADATA_SIZE) & 1350 ~(SQUASHFS_METADATA_SIZE - 1)) : dir_size - data_space; 1351 1352 void *dc = realloc(directory_data_cache, 1353 directory_cache_size + realloc_size); 1354 if(dc == NULL) 1355 MEM_ERROR(); 1356 directory_cache_size += realloc_size; 1357 directory_data_cache = dc; 1358 } 1359 1360 if(dir_size) { 1361 struct squashfs_dir_header dir_header; 1362 1363 dir_header.count = dir->entry_count - 1; 1364 dir_header.start_block = dir->start_block; 1365 dir_header.inode_number = dir->inode_number; 1366 SQUASHFS_SWAP_DIR_HEADER(&dir_header, dir->entry_count_p); 1367 memcpy(directory_data_cache + directory_cache_bytes, dir->buff, 1368 dir_size); 1369 } 1370 directory_offset = directory_cache_bytes; 1371 directory_block = directory_bytes; 1372 directory_cache_bytes += dir_size; 1373 i_count = 0; 1374 index = SQUASHFS_METADATA_SIZE - directory_offset; 1375 1376 while(1) { 1377 while(i_count < dir->i_count && 1378 dir->index[i_count].index.index < index) 1379 dir->index[i_count++].index.start_block = 1380 directory_bytes; 1381 index += SQUASHFS_METADATA_SIZE; 1382 1383 if(directory_cache_bytes < SQUASHFS_METADATA_SIZE) 1384 break; 1385 1386 if((directory_size - directory_bytes) < 1387 ((SQUASHFS_METADATA_SIZE << 1) + 2)) { 1388 void *dt = realloc(directory_table, 1389 directory_size + (SQUASHFS_METADATA_SIZE << 1) 1390 + 2); 1391 if(dt == NULL) 1392 MEM_ERROR(); 1393 directory_size += SQUASHFS_METADATA_SIZE << 1; 1394 directory_table = dt; 1395 } 1396 1397 c_byte = mangle(directory_table + directory_bytes + 1398 BLOCK_OFFSET, directory_data_cache, 1399 SQUASHFS_METADATA_SIZE, SQUASHFS_METADATA_SIZE, 1400 noI, 0); 1401 TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, 1402 c_byte); 1403 SQUASHFS_SWAP_SHORTS(&c_byte, 1404 directory_table + directory_bytes, 1); 1405 directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + 1406 BLOCK_OFFSET; 1407 total_directory_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; 1408 memmove(directory_data_cache, directory_data_cache + 1409 SQUASHFS_METADATA_SIZE, directory_cache_bytes - 1410 SQUASHFS_METADATA_SIZE); 1411 directory_cache_bytes -= SQUASHFS_METADATA_SIZE; 1412 } 1413 1414 create_inode(inode, dir_info, dir_info->dir_ent, SQUASHFS_DIR_TYPE, 1415 dir_size + 3, directory_block, directory_offset, NULL, NULL, 1416 dir, 0); 1417 1418 #ifdef SQUASHFS_TRACE 1419 { 1420 unsigned char *dirp; 1421 int count; 1422 1423 TRACE("Directory contents of inode 0x%llx\n", *inode); 1424 dirp = dir->buff; 1425 while(dirp < dir->p) { 1426 char buffer[SQUASHFS_NAME_LEN + 1]; 1427 struct squashfs_dir_entry idir, *idirp; 1428 struct squashfs_dir_header dirh; 1429 SQUASHFS_SWAP_DIR_HEADER((struct squashfs_dir_header *) dirp, 1430 &dirh); 1431 count = dirh.count + 1; 1432 dirp += sizeof(struct squashfs_dir_header); 1433 1434 TRACE("\tStart block 0x%x, count %d\n", 1435 dirh.start_block, count); 1436 1437 while(count--) { 1438 idirp = (struct squashfs_dir_entry *) dirp; 1439 SQUASHFS_SWAP_DIR_ENTRY(idirp, &idir); 1440 strncpy(buffer, idirp->name, idir.size + 1); 1441 buffer[idir.size + 1] = '\0'; 1442 TRACE("\t\tname %s, inode offset 0x%x, type " 1443 "%d\n", buffer, idir.offset, idir.type); 1444 dirp += sizeof(struct squashfs_dir_entry) + idir.size + 1445 1; 1446 } 1447 } 1448 } 1449 #endif 1450 dir_count ++; 1451 } 1452 1453 1454 static struct file_buffer *get_fragment(struct fragment *fragment) 1455 { 1456 struct squashfs_fragment_entry *disk_fragment; 1457 struct file_buffer *buffer, *compressed_buffer; 1458 long long start_block; 1459 int res, size, index = fragment->index; 1460 char locked; 1461 1462 /* 1463 * Lookup fragment block in cache. 1464 * If the fragment block doesn't exist, then get the compressed version 1465 * from the writer cache or off disk, and decompress it. 1466 * 1467 * This routine has two things which complicate the code: 1468 * 1469 * 1. Multiple threads can simultaneously lookup/create the 1470 * same buffer. This means a buffer needs to be "locked" 1471 * when it is being filled in, to prevent other threads from 1472 * using it when it is not ready. This is because we now do 1473 * fragment duplicate checking in parallel. 1474 * 2. We have two caches which need to be checked for the 1475 * presence of fragment blocks: the normal fragment cache 1476 * and a "reserve" cache. The reserve cache is used to 1477 * prevent an unnecessary pipeline stall when the fragment cache 1478 * is full of fragments waiting to be compressed. 1479 */ 1480 1481 if(fragment->index == SQUASHFS_INVALID_FRAG) 1482 return NULL; 1483 1484 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1485 pthread_mutex_lock(&dup_mutex); 1486 1487 again: 1488 buffer = cache_lookup_nowait(fragment_buffer, index, &locked); 1489 if(buffer) { 1490 pthread_mutex_unlock(&dup_mutex); 1491 if(locked) 1492 /* got a buffer being filled in. Wait for it */ 1493 cache_wait_unlock(buffer); 1494 goto finished; 1495 } 1496 1497 /* not in fragment cache, is it in the reserve cache? */ 1498 buffer = cache_lookup_nowait(reserve_cache, index, &locked); 1499 if(buffer) { 1500 pthread_mutex_unlock(&dup_mutex); 1501 if(locked) 1502 /* got a buffer being filled in. Wait for it */ 1503 cache_wait_unlock(buffer); 1504 goto finished; 1505 } 1506 1507 /* in neither cache, try to get it from the fragment cache */ 1508 buffer = cache_get_nowait(fragment_buffer, index); 1509 if(!buffer) { 1510 /* 1511 * no room, get it from the reserve cache, this is 1512 * dimensioned so it will always have space (no more than 1513 * processors + 1 can have an outstanding reserve buffer) 1514 */ 1515 buffer = cache_get_nowait(reserve_cache, index); 1516 if(!buffer) { 1517 /* failsafe */ 1518 ERROR("no space in reserve cache\n"); 1519 goto again; 1520 } 1521 } 1522 1523 pthread_mutex_unlock(&dup_mutex); 1524 1525 compressed_buffer = cache_lookup(fwriter_buffer, index); 1526 1527 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1528 pthread_mutex_lock(&fragment_mutex); 1529 disk_fragment = &fragment_table[index]; 1530 size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size); 1531 start_block = disk_fragment->start_block; 1532 pthread_cleanup_pop(1); 1533 1534 if(SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) { 1535 int error; 1536 char *data; 1537 1538 if(compressed_buffer) 1539 data = compressed_buffer->data; 1540 else { 1541 data = read_from_disk(start_block, size); 1542 if(data == NULL) { 1543 ERROR("Failed to read fragment from output" 1544 " filesystem\n"); 1545 BAD_ERROR("Output filesystem corrupted?\n"); 1546 } 1547 } 1548 1549 res = compressor_uncompress(comp, buffer->data, data, size, 1550 block_size, &error); 1551 if(res == -1) 1552 BAD_ERROR("%s uncompress failed with error code %d\n", 1553 comp->name, error); 1554 } else if(compressed_buffer) 1555 memcpy(buffer->data, compressed_buffer->data, size); 1556 else { 1557 res = read_fs_bytes(fd, start_block, size, buffer->data); 1558 if(res == 0) { 1559 ERROR("Failed to read fragment from output " 1560 "filesystem\n"); 1561 BAD_ERROR("Output filesystem corrupted?\n"); 1562 } 1563 } 1564 1565 cache_unlock(buffer); 1566 cache_block_put(compressed_buffer); 1567 1568 finished: 1569 pthread_cleanup_pop(0); 1570 1571 return buffer; 1572 } 1573 1574 1575 unsigned short get_fragment_checksum(struct file_info *file) 1576 { 1577 struct file_buffer *frag_buffer; 1578 struct append_file *append; 1579 int res, index = file->fragment->index; 1580 unsigned short checksum; 1581 1582 if(index == SQUASHFS_INVALID_FRAG) 1583 return 0; 1584 1585 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1586 pthread_mutex_lock(&dup_mutex); 1587 res = file->have_frag_checksum; 1588 checksum = file->fragment_checksum; 1589 pthread_cleanup_pop(1); 1590 1591 if(res) 1592 return checksum; 1593 1594 frag_buffer = get_fragment(file->fragment); 1595 1596 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1597 1598 for(append = file_mapping[index]; append; append = append->next) { 1599 int offset = append->file->fragment->offset; 1600 int size = append->file->fragment->size; 1601 unsigned short cksum = 1602 get_checksum_mem(frag_buffer->data + offset, size); 1603 1604 if(file == append->file) 1605 checksum = cksum; 1606 1607 pthread_mutex_lock(&dup_mutex); 1608 append->file->fragment_checksum = cksum; 1609 append->file->have_frag_checksum = TRUE; 1610 pthread_mutex_unlock(&dup_mutex); 1611 } 1612 1613 cache_block_put(frag_buffer); 1614 pthread_cleanup_pop(0); 1615 1616 return checksum; 1617 } 1618 1619 1620 void lock_fragments() 1621 { 1622 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1623 pthread_mutex_lock(&fragment_mutex); 1624 fragments_locked = TRUE; 1625 pthread_cleanup_pop(1); 1626 } 1627 1628 1629 void unlock_fragments() 1630 { 1631 int frg, size; 1632 struct file_buffer *write_buffer; 1633 1634 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1635 pthread_mutex_lock(&fragment_mutex); 1636 1637 /* 1638 * Note queue_empty() is inherently racy with respect to concurrent 1639 * queue get and pushes. We avoid this because we're holding the 1640 * fragment_mutex which ensures no other threads can be using the 1641 * queue at this time. 1642 */ 1643 while(!queue_empty(locked_fragment)) { 1644 write_buffer = queue_get(locked_fragment); 1645 frg = write_buffer->block; 1646 size = SQUASHFS_COMPRESSED_SIZE_BLOCK(fragment_table[frg].size); 1647 fragment_table[frg].start_block = bytes; 1648 write_buffer->block = bytes; 1649 bytes += size; 1650 fragments_outstanding --; 1651 queue_put(to_writer, write_buffer); 1652 TRACE("fragment_locked writing fragment %d, compressed size %d" 1653 "\n", frg, size); 1654 } 1655 fragments_locked = FALSE; 1656 pthread_cleanup_pop(1); 1657 } 1658 1659 /* Called with the fragment_mutex locked */ 1660 void add_pending_fragment(struct file_buffer *write_buffer, int c_byte, 1661 int fragment) 1662 { 1663 fragment_table[fragment].size = c_byte; 1664 write_buffer->block = fragment; 1665 1666 queue_put(locked_fragment, write_buffer); 1667 } 1668 1669 1670 void write_fragment(struct file_buffer *fragment) 1671 { 1672 if(fragment == NULL) 1673 return; 1674 1675 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1676 pthread_mutex_lock(&fragment_mutex); 1677 fragment_table[fragment->block].unused = 0; 1678 fragments_outstanding ++; 1679 queue_put(to_frag, fragment); 1680 pthread_cleanup_pop(1); 1681 } 1682 1683 1684 struct file_buffer *allocate_fragment() 1685 { 1686 struct file_buffer *fragment = cache_get(fragment_buffer, fragments); 1687 1688 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1689 pthread_mutex_lock(&fragment_mutex); 1690 1691 if(fragments % FRAG_SIZE == 0) { 1692 void *ft = realloc(fragment_table, (fragments + 1693 FRAG_SIZE) * sizeof(struct squashfs_fragment_entry)); 1694 if(ft == NULL) 1695 MEM_ERROR(); 1696 fragment_table = ft; 1697 } 1698 1699 fragment->size = 0; 1700 fragment->block = fragments ++; 1701 1702 pthread_cleanup_pop(1); 1703 1704 return fragment; 1705 } 1706 1707 1708 static struct fragment empty_fragment = {SQUASHFS_INVALID_FRAG, 0, 0}; 1709 1710 1711 void free_fragment(struct fragment *fragment) 1712 { 1713 if(fragment != &empty_fragment) 1714 free(fragment); 1715 } 1716 1717 1718 struct fragment *get_and_fill_fragment(struct file_buffer *file_buffer, 1719 struct dir_ent *dir_ent) 1720 { 1721 struct fragment *ffrg; 1722 struct file_buffer **fragment; 1723 1724 if(file_buffer == NULL || file_buffer->size == 0) 1725 return &empty_fragment; 1726 1727 fragment = eval_frag_actions(root_dir, dir_ent); 1728 1729 if((*fragment) && (*fragment)->size + file_buffer->size > block_size) { 1730 write_fragment(*fragment); 1731 *fragment = NULL; 1732 } 1733 1734 ffrg = malloc(sizeof(struct fragment)); 1735 if(ffrg == NULL) 1736 MEM_ERROR(); 1737 1738 if(*fragment == NULL) 1739 *fragment = allocate_fragment(); 1740 1741 ffrg->index = (*fragment)->block; 1742 ffrg->offset = (*fragment)->size; 1743 ffrg->size = file_buffer->size; 1744 memcpy((*fragment)->data + (*fragment)->size, file_buffer->data, 1745 file_buffer->size); 1746 (*fragment)->size += file_buffer->size; 1747 1748 return ffrg; 1749 } 1750 1751 1752 long long generic_write_table(int length, void *buffer, int length2, 1753 void *buffer2, int uncompressed) 1754 { 1755 int meta_blocks = (length + SQUASHFS_METADATA_SIZE - 1) / 1756 SQUASHFS_METADATA_SIZE; 1757 long long *list, start_bytes; 1758 int compressed_size, i, list_size = meta_blocks * sizeof(long long); 1759 unsigned short c_byte; 1760 char cbuffer[(SQUASHFS_METADATA_SIZE << 2) + 2]; 1761 1762 #ifdef SQUASHFS_TRACE 1763 long long obytes = bytes; 1764 int olength = length; 1765 #endif 1766 1767 list = malloc(list_size); 1768 if(list == NULL) 1769 MEM_ERROR(); 1770 1771 for(i = 0; i < meta_blocks; i++) { 1772 int avail_bytes = length > SQUASHFS_METADATA_SIZE ? 1773 SQUASHFS_METADATA_SIZE : length; 1774 c_byte = mangle(cbuffer + BLOCK_OFFSET, buffer + i * 1775 SQUASHFS_METADATA_SIZE , avail_bytes, 1776 SQUASHFS_METADATA_SIZE, uncompressed, 0); 1777 SQUASHFS_SWAP_SHORTS(&c_byte, cbuffer, 1); 1778 list[i] = bytes; 1779 compressed_size = SQUASHFS_COMPRESSED_SIZE(c_byte) + 1780 BLOCK_OFFSET; 1781 TRACE("block %d @ 0x%llx, compressed size %d\n", i, bytes, 1782 compressed_size); 1783 write_destination(fd, bytes, compressed_size, cbuffer); 1784 bytes += compressed_size; 1785 total_bytes += avail_bytes; 1786 length -= avail_bytes; 1787 } 1788 1789 start_bytes = bytes; 1790 if(length2) { 1791 write_destination(fd, bytes, length2, buffer2); 1792 bytes += length2; 1793 total_bytes += length2; 1794 } 1795 1796 SQUASHFS_INSWAP_LONG_LONGS(list, meta_blocks); 1797 write_destination(fd, bytes, list_size, list); 1798 bytes += list_size; 1799 total_bytes += list_size; 1800 1801 TRACE("generic_write_table: total uncompressed %d compressed %lld\n", 1802 olength, bytes - obytes); 1803 1804 free(list); 1805 1806 return start_bytes; 1807 } 1808 1809 1810 long long write_fragment_table() 1811 { 1812 unsigned int frag_bytes = SQUASHFS_FRAGMENT_BYTES(fragments); 1813 int i; 1814 1815 TRACE("write_fragment_table: fragments %d, frag_bytes %d\n", fragments, 1816 frag_bytes); 1817 for(i = 0; i < fragments; i++) { 1818 TRACE("write_fragment_table: fragment %d, start_block 0x%llx, " 1819 "size %d\n", i, fragment_table[i].start_block, 1820 fragment_table[i].size); 1821 SQUASHFS_INSWAP_FRAGMENT_ENTRY(&fragment_table[i]); 1822 } 1823 1824 return generic_write_table(frag_bytes, fragment_table, 0, NULL, noF); 1825 } 1826 1827 1828 char read_from_file_buffer[SQUASHFS_FILE_MAX_SIZE]; 1829 static char *read_from_disk(long long start, unsigned int avail_bytes) 1830 { 1831 int res; 1832 1833 res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer); 1834 if(res == 0) 1835 return NULL; 1836 1837 return read_from_file_buffer; 1838 } 1839 1840 1841 char read_from_file_buffer2[SQUASHFS_FILE_MAX_SIZE]; 1842 char *read_from_disk2(long long start, unsigned int avail_bytes) 1843 { 1844 int res; 1845 1846 res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer2); 1847 if(res == 0) 1848 return NULL; 1849 1850 return read_from_file_buffer2; 1851 } 1852 1853 1854 /* 1855 * Compute 16 bit BSD checksum over the data 1856 */ 1857 unsigned short get_checksum(char *buff, int bytes, unsigned short chksum) 1858 { 1859 unsigned char *b = (unsigned char *) buff; 1860 1861 while(bytes --) { 1862 chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1; 1863 chksum += *b++; 1864 } 1865 1866 return chksum; 1867 } 1868 1869 1870 unsigned short get_checksum_disk(long long start, long long l, 1871 unsigned int *blocks) 1872 { 1873 unsigned short chksum = 0; 1874 unsigned int bytes; 1875 struct file_buffer *write_buffer; 1876 int i; 1877 1878 for(i = 0; l; i++) { 1879 bytes = SQUASHFS_COMPRESSED_SIZE_BLOCK(blocks[i]); 1880 if(bytes == 0) /* sparse block */ 1881 continue; 1882 write_buffer = cache_lookup(bwriter_buffer, start); 1883 if(write_buffer) { 1884 chksum = get_checksum(write_buffer->data, bytes, 1885 chksum); 1886 cache_block_put(write_buffer); 1887 } else { 1888 void *data = read_from_disk(start, bytes); 1889 if(data == NULL) { 1890 ERROR("Failed to checksum data from output" 1891 " filesystem\n"); 1892 BAD_ERROR("Output filesystem corrupted?\n"); 1893 } 1894 1895 chksum = get_checksum(data, bytes, chksum); 1896 } 1897 1898 l -= bytes; 1899 start += bytes; 1900 } 1901 1902 return chksum; 1903 } 1904 1905 1906 unsigned short get_checksum_mem(char *buff, int bytes) 1907 { 1908 return get_checksum(buff, bytes, 0); 1909 } 1910 1911 1912 unsigned short get_checksum_mem_buffer(struct file_buffer *file_buffer) 1913 { 1914 if(file_buffer == NULL) 1915 return 0; 1916 else 1917 return get_checksum(file_buffer->data, file_buffer->size, 0); 1918 } 1919 1920 1921 #define DUP_HASH(a) (a & 0xffff) 1922 void add_file(long long start, long long file_size, long long file_bytes, 1923 unsigned int *block_listp, int blocks, unsigned int fragment, 1924 int offset, int bytes) 1925 { 1926 struct fragment *frg; 1927 unsigned int *block_list = block_listp; 1928 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1929 struct append_file *append_file; 1930 struct file_info *file; 1931 1932 if(!duplicate_checking || file_size == 0) 1933 return; 1934 1935 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) { 1936 if(file_size != dupl_ptr->file_size) 1937 continue; 1938 if(blocks != 0 && start != dupl_ptr->start) 1939 continue; 1940 if(fragment != dupl_ptr->fragment->index) 1941 continue; 1942 if(fragment != SQUASHFS_INVALID_FRAG && (offset != 1943 dupl_ptr->fragment->offset || bytes != 1944 dupl_ptr->fragment->size)) 1945 continue; 1946 return; 1947 } 1948 1949 frg = malloc(sizeof(struct fragment)); 1950 if(frg == NULL) 1951 MEM_ERROR(); 1952 1953 frg->index = fragment; 1954 frg->offset = offset; 1955 frg->size = bytes; 1956 1957 file = add_non_dup(file_size, file_bytes, block_list, start, frg, 0, 0, 1958 FALSE, FALSE); 1959 1960 if(fragment == SQUASHFS_INVALID_FRAG) 1961 return; 1962 1963 append_file = malloc(sizeof(struct append_file)); 1964 if(append_file == NULL) 1965 MEM_ERROR(); 1966 1967 append_file->file = file; 1968 append_file->next = file_mapping[fragment]; 1969 file_mapping[fragment] = append_file; 1970 } 1971 1972 1973 int pre_duplicate(long long file_size) 1974 { 1975 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 1976 1977 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) 1978 if(dupl_ptr->file_size == file_size) 1979 return TRUE; 1980 1981 return FALSE; 1982 } 1983 1984 1985 struct file_info *add_non_dup(long long file_size, long long bytes, 1986 unsigned int *block_list, long long start, struct fragment *fragment, 1987 unsigned short checksum, unsigned short fragment_checksum, 1988 int checksum_flag, int checksum_frag_flag) 1989 { 1990 struct file_info *dupl_ptr = malloc(sizeof(struct file_info)); 1991 1992 if(dupl_ptr == NULL) 1993 MEM_ERROR(); 1994 1995 dupl_ptr->file_size = file_size; 1996 dupl_ptr->bytes = bytes; 1997 dupl_ptr->block_list = block_list; 1998 dupl_ptr->start = start; 1999 dupl_ptr->fragment = fragment; 2000 dupl_ptr->checksum = checksum; 2001 dupl_ptr->fragment_checksum = fragment_checksum; 2002 dupl_ptr->have_frag_checksum = checksum_frag_flag; 2003 dupl_ptr->have_checksum = checksum_flag; 2004 2005 pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 2006 pthread_mutex_lock(&dup_mutex); 2007 dupl_ptr->next = dupl[DUP_HASH(file_size)]; 2008 dupl[DUP_HASH(file_size)] = dupl_ptr; 2009 dup_files ++; 2010 pthread_cleanup_pop(1); 2011 2012 return dupl_ptr; 2013 } 2014 2015 2016 struct fragment *frag_duplicate(struct file_buffer *file_buffer, char *dont_put) 2017 { 2018 struct file_info *dupl_ptr; 2019 struct file_buffer *buffer; 2020 struct file_info *dupl_start = file_buffer->dupl_start; 2021 long long file_size = file_buffer->file_size; 2022 unsigned short checksum = file_buffer->checksum; 2023 int res; 2024 2025 if(file_buffer->duplicate) { 2026 TRACE("Found duplicate file, fragment %d, size %d, offset %d, " 2027 "checksum 0x%x\n", dupl_start->fragment->index, 2028 file_size, dupl_start->fragment->offset, checksum); 2029 *dont_put = TRUE; 2030 return dupl_start->fragment; 2031 } else { 2032 *dont_put = FALSE; 2033 dupl_ptr = dupl[DUP_HASH(file_size)]; 2034 } 2035 2036 for(; dupl_ptr && dupl_ptr != dupl_start; dupl_ptr = dupl_ptr->next) { 2037 if(file_size == dupl_ptr->file_size && file_size == 2038 dupl_ptr->fragment->size) { 2039 if(get_fragment_checksum(dupl_ptr) == checksum) { 2040 buffer = get_fragment(dupl_ptr->fragment); 2041 res = memcmp(file_buffer->data, buffer->data + 2042 dupl_ptr->fragment->offset, file_size); 2043 cache_block_put(buffer); 2044 if(res == 0) 2045 break; 2046 } 2047 } 2048 } 2049 2050 if(!dupl_ptr || dupl_ptr == dupl_start) 2051 return NULL; 2052 2053 TRACE("Found duplicate file, fragment %d, size %d, offset %d, " 2054 "checksum 0x%x\n", dupl_ptr->fragment->index, file_size, 2055 dupl_ptr->fragment->offset, checksum); 2056 2057 return dupl_ptr->fragment; 2058 } 2059 2060 2061 struct file_info *duplicate(long long file_size, long long bytes, 2062 unsigned int **block_list, long long *start, struct fragment **fragment, 2063 struct file_buffer *file_buffer, int blocks, unsigned short checksum, 2064 int checksum_flag) 2065 { 2066 struct file_info *dupl_ptr = dupl[DUP_HASH(file_size)]; 2067 int frag_bytes = file_buffer ? file_buffer->size : 0; 2068 unsigned short fragment_checksum = file_buffer ? 2069 file_buffer->checksum : 0; 2070 2071 for(; dupl_ptr; dupl_ptr = dupl_ptr->next) 2072 if(file_size == dupl_ptr->file_size && bytes == dupl_ptr->bytes 2073 && frag_bytes == dupl_ptr->fragment->size) { 2074 long long target_start, dup_start = dupl_ptr->start; 2075 int block; 2076 2077 if(memcmp(*block_list, dupl_ptr->block_list, blocks * 2078 sizeof(unsigned int)) != 0) 2079 continue; 2080 2081 if(checksum_flag == FALSE) { 2082 checksum = get_checksum_disk(*start, bytes, 2083 *block_list); 2084 checksum_flag = TRUE; 2085 } 2086 2087 if(!dupl_ptr->have_checksum) { 2088 dupl_ptr->checksum = 2089 get_checksum_disk(dupl_ptr->start, 2090 dupl_ptr->bytes, dupl_ptr->block_list); 2091 dupl_ptr->have_checksum = TRUE; 2092 } 2093 2094 if(checksum != dupl_ptr->checksum || 2095 fragment_checksum != 2096 get_fragment_checksum(dupl_ptr)) 2097 continue; 2098 2099 target_start = *start; 2100 for(block = 0; block < blocks; block ++) { 2101 int size = SQUASHFS_COMPRESSED_SIZE_BLOCK 2102 ((*block_list)[block]); 2103 struct file_buffer *target_buffer = NULL; 2104 struct file_buffer *dup_buffer = NULL; 2105 char *target_data, *dup_data; 2106 int res; 2107 2108 if(size == 0) 2109 continue; 2110 target_buffer = cache_lookup(bwriter_buffer, 2111 target_start); 2112 if(target_buffer) 2113 target_data = target_buffer->data; 2114 else { 2115 target_data = 2116 read_from_disk(target_start, 2117 size); 2118 if(target_data == NULL) { 2119 ERROR("Failed to read data from" 2120 " output filesystem\n"); 2121 BAD_ERROR("Output filesystem" 2122 " corrupted?\n"); 2123 } 2124 } 2125 2126 dup_buffer = cache_lookup(bwriter_buffer, 2127 dup_start); 2128 if(dup_buffer) 2129 dup_data = dup_buffer->data; 2130 else { 2131 dup_data = read_from_disk2(dup_start, 2132 size); 2133 if(dup_data == NULL) { 2134 ERROR("Failed to read data from" 2135 " output filesystem\n"); 2136 BAD_ERROR("Output filesystem" 2137 " corrupted?\n"); 2138 } 2139 } 2140 2141 res = memcmp(target_data, dup_data, size); 2142 cache_block_put(target_buffer); 2143 cache_block_put(dup_buffer); 2144 if(res != 0) 2145 break; 2146 target_start += size; 2147 dup_start += size; 2148 } 2149 if(block == blocks) { 2150 struct file_buffer *frag_buffer = 2151 get_fragment(dupl_ptr->fragment); 2152 2153 if(frag_bytes == 0 || 2154 memcmp(file_buffer->data, 2155 frag_buffer->data + 2156 dupl_ptr->fragment->offset, 2157 frag_bytes) == 0) { 2158 TRACE("Found duplicate file, start " 2159 "0x%llx, size %lld, checksum " 2160 "0x%x, fragment %d, size %d, " 2161 "offset %d, checksum 0x%x\n", 2162 dupl_ptr->start, 2163 dupl_ptr->bytes, 2164 dupl_ptr->checksum, 2165 dupl_ptr->fragment->index, 2166 frag_bytes, 2167 dupl_ptr->fragment->offset, 2168 fragment_checksum); 2169 *block_list = dupl_ptr->block_list; 2170 *start = dupl_ptr->start; 2171 *fragment = dupl_ptr->fragment; 2172 cache_block_put(frag_buffer); 2173 return 0; 2174 } 2175 cache_block_put(frag_buffer); 2176 } 2177 } 2178 2179 2180 return add_non_dup(file_size, bytes, *block_list, *start, *fragment, 2181 checksum, fragment_checksum, checksum_flag, TRUE); 2182 } 2183 2184 2185 static inline int is_fragment(struct inode_info *inode) 2186 { 2187 off_t file_size = inode->buf.st_size; 2188 2189 /* 2190 * If this block is to be compressed differently to the 2191 * fragment compression then it cannot be a fragment 2192 */ 2193 if(inode->noF != noF) 2194 return FALSE; 2195 2196 return !inode->no_fragments && file_size && (file_size < block_size || 2197 (inode->always_use_fragments && file_size & (block_size - 1))); 2198 } 2199 2200 2201 void put_file_buffer(struct file_buffer *file_buffer) 2202 { 2203 /* 2204 * Decide where to send the file buffer: 2205 * - compressible non-fragment blocks go to the deflate threads, 2206 * - fragments go to the process fragment threads, 2207 * - all others go directly to the main thread 2208 */ 2209 if(file_buffer->error) { 2210 file_buffer->fragment = 0; 2211 seq_queue_put(to_main, file_buffer); 2212 } else if (file_buffer->file_size == 0) 2213 seq_queue_put(to_main, file_buffer); 2214 else if(file_buffer->fragment) 2215 queue_put(to_process_frag, file_buffer); 2216 else 2217 queue_put(to_deflate, file_buffer); 2218 } 2219 2220 2221 static int seq = 0; 2222 void reader_read_process(struct dir_ent *dir_ent) 2223 { 2224 long long bytes = 0; 2225 struct inode_info *inode = dir_ent->inode; 2226 struct file_buffer *prev_buffer = NULL, *file_buffer; 2227 int status, byte, res, child; 2228 int file = pseudo_exec_file(get_pseudo_file(inode->pseudo_id), &child); 2229 2230 if(!file) { 2231 file_buffer = cache_get_nohash(reader_buffer); 2232 file_buffer->sequence = seq ++; 2233 goto read_err; 2234 } 2235 2236 while(1) { 2237 file_buffer = cache_get_nohash(reader_buffer); 2238 file_buffer->sequence = seq ++; 2239 file_buffer->noD = inode->noD; 2240 2241 byte = read_bytes(file, file_buffer->data, block_size); 2242 if(byte == -1) 2243 goto read_err2; 2244 2245 file_buffer->size = byte; 2246 file_buffer->file_size = -1; 2247 file_buffer->error = FALSE; 2248 file_buffer->fragment = FALSE; 2249 bytes += byte; 2250 2251 if(byte == 0) 2252 break; 2253 2254 /* 2255 * Update progress bar size. This is done 2256 * on every block rather than waiting for all blocks to be 2257 * read incase write_file_process() is running in parallel 2258 * with this. Otherwise the current progress bar position 2259 * may get ahead of the progress bar size. 2260 */ 2261 progress_bar_size(1); 2262 2263 if(prev_buffer) 2264 put_file_buffer(prev_buffer); 2265 prev_buffer = file_buffer; 2266 } 2267 2268 /* 2269 * Update inode file size now that the size of the dynamic pseudo file 2270 * is known. This is needed for the -info option. 2271 */ 2272 inode->buf.st_size = bytes; 2273 2274 res = waitpid(child, &status, 0); 2275 close(file); 2276 2277 if(res == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) 2278 goto read_err; 2279 2280 if(prev_buffer == NULL) 2281 prev_buffer = file_buffer; 2282 else { 2283 cache_block_put(file_buffer); 2284 seq --; 2285 } 2286 prev_buffer->file_size = bytes; 2287 prev_buffer->fragment = is_fragment(inode); 2288 put_file_buffer(prev_buffer); 2289 2290 return; 2291 2292 read_err2: 2293 close(file); 2294 read_err: 2295 if(prev_buffer) { 2296 cache_block_put(file_buffer); 2297 seq --; 2298 file_buffer = prev_buffer; 2299 } 2300 file_buffer->error = TRUE; 2301 put_file_buffer(file_buffer); 2302 } 2303 2304 2305 void reader_read_file(struct dir_ent *dir_ent) 2306 { 2307 struct stat *buf = &dir_ent->inode->buf, buf2; 2308 struct file_buffer *file_buffer; 2309 int blocks, file, res; 2310 long long bytes, read_size; 2311 struct inode_info *inode = dir_ent->inode; 2312 2313 if(inode->read) 2314 return; 2315 2316 inode->read = TRUE; 2317 again: 2318 bytes = 0; 2319 read_size = buf->st_size; 2320 blocks = (read_size + block_size - 1) >> block_log; 2321 2322 file = open(pathname_reader(dir_ent), O_RDONLY); 2323 if(file == -1) { 2324 file_buffer = cache_get_nohash(reader_buffer); 2325 file_buffer->sequence = seq ++; 2326 goto read_err2; 2327 } 2328 2329 do { 2330 file_buffer = cache_get_nohash(reader_buffer); 2331 file_buffer->file_size = read_size; 2332 file_buffer->sequence = seq ++; 2333 file_buffer->noD = inode->noD; 2334 file_buffer->error = FALSE; 2335 2336 /* 2337 * Always try to read block_size bytes from the file rather 2338 * than expected bytes (which will be less than the block_size 2339 * at the file tail) to check that the file hasn't grown 2340 * since being stated. If it is longer (or shorter) than 2341 * expected, then restat, and try again. Note the special 2342 * case where the file is an exact multiple of the block_size 2343 * is dealt with later. 2344 */ 2345 file_buffer->size = read_bytes(file, file_buffer->data, 2346 block_size); 2347 if(file_buffer->size == -1) 2348 goto read_err; 2349 2350 bytes += file_buffer->size; 2351 2352 if(blocks > 1) { 2353 /* non-tail block should be exactly block_size */ 2354 if(file_buffer->size < block_size) 2355 goto restat; 2356 2357 file_buffer->fragment = FALSE; 2358 put_file_buffer(file_buffer); 2359 } 2360 } while(-- blocks > 0); 2361 2362 /* Overall size including tail should match */ 2363 if(read_size != bytes) 2364 goto restat; 2365 2366 if(read_size && read_size % block_size == 0) { 2367 /* 2368 * Special case where we've not tried to read past the end of 2369 * the file. We expect to get EOF, i.e. the file isn't larger 2370 * than we expect. 2371 */ 2372 char buffer; 2373 int res; 2374 2375 res = read_bytes(file, &buffer, 1); 2376 if(res == -1) 2377 goto read_err; 2378 2379 if(res != 0) 2380 goto restat; 2381 } 2382 2383 file_buffer->fragment = is_fragment(inode); 2384 put_file_buffer(file_buffer); 2385 2386 close(file); 2387 2388 return; 2389 2390 restat: 2391 res = fstat(file, &buf2); 2392 if(res == -1) { 2393 ERROR("Cannot stat dir/file %s because %s\n", 2394 pathname_reader(dir_ent), strerror(errno)); 2395 goto read_err; 2396 } 2397 2398 if(read_size != buf2.st_size) { 2399 close(file); 2400 memcpy(buf, &buf2, sizeof(struct stat)); 2401 file_buffer->error = 2; 2402 put_file_buffer(file_buffer); 2403 goto again; 2404 } 2405 read_err: 2406 close(file); 2407 read_err2: 2408 file_buffer->error = TRUE; 2409 put_file_buffer(file_buffer); 2410 } 2411 2412 2413 void reader_scan(struct dir_info *dir) { 2414 struct dir_ent *dir_ent = dir->list; 2415 2416 for(; dir_ent; dir_ent = dir_ent->next) { 2417 struct stat *buf = &dir_ent->inode->buf; 2418 if(dir_ent->inode->root_entry) 2419 continue; 2420 2421 if(IS_PSEUDO_PROCESS(dir_ent->inode)) { 2422 reader_read_process(dir_ent); 2423 continue; 2424 } 2425 2426 switch(buf->st_mode & S_IFMT) { 2427 case S_IFREG: 2428 reader_read_file(dir_ent); 2429 break; 2430 case S_IFDIR: 2431 reader_scan(dir_ent->dir); 2432 break; 2433 } 2434 } 2435 } 2436 2437 2438 void *reader(void *arg) 2439 { 2440 if(!sorted) 2441 reader_scan(queue_get(to_reader)); 2442 else { 2443 int i; 2444 struct priority_entry *entry; 2445 2446 queue_get(to_reader); 2447 for(i = 65535; i >= 0; i--) 2448 for(entry = priority_list[i]; entry; 2449 entry = entry->next) 2450 reader_read_file(entry->dir); 2451 } 2452 2453 pthread_exit(NULL); 2454 } 2455 2456 2457 void *writer(void *arg) 2458 { 2459 while(1) { 2460 struct file_buffer *file_buffer = queue_get(to_writer); 2461 off_t off; 2462 2463 if(file_buffer == NULL) { 2464 queue_put(from_writer, NULL); 2465 continue; 2466 } 2467 2468 off = file_buffer->block; 2469 2470 pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); 2471 pthread_mutex_lock(&pos_mutex); 2472 2473 if(lseek(fd, off, SEEK_SET) == -1) { 2474 ERROR("writer: Lseek on destination failed because " 2475 "%s, offset=0x%llx\n", strerror(errno), off); 2476 BAD_ERROR("Probably out of space on output " 2477 "%s\n", block_device ? "block device" : 2478 "filesystem"); 2479 } 2480 2481 if(write_bytes(fd, file_buffer->data, 2482 file_buffer->size) == -1) 2483 BAD_ERROR("Failed to write to output %s\n", 2484 block_device ? "block device" : "filesystem"); 2485 2486 pthread_cleanup_pop(1); 2487 2488 cache_block_put(file_buffer); 2489 } 2490 } 2491 2492 2493 int all_zero(struct file_buffer *file_buffer) 2494 { 2495 int i; 2496 long entries = file_buffer->size / sizeof(long); 2497 long *p = (long *) file_buffer->data; 2498 2499 for(i = 0; i < entries && p[i] == 0; i++); 2500 2501 if(i == entries) { 2502 for(i = file_buffer->size & ~(sizeof(long) - 1); 2503 i < file_buffer->size && file_buffer->data[i] == 0; 2504 i++); 2505 2506 return i == file_buffer->size; 2507 } 2508 2509 return 0; 2510 } 2511 2512 2513 void *deflator(void *arg) 2514 { 2515 struct file_buffer *write_buffer = cache_get_nohash(bwriter_buffer); 2516 void *stream = NULL; 2517 int res; 2518 2519 res = compressor_init(comp, &stream, block_size, 1); 2520 if(res) 2521 BAD_ERROR("deflator:: compressor_init failed\n"); 2522 2523 while(1) { 2524 struct file_buffer *file_buffer = queue_get(to_deflate); 2525 2526 if(sparse_files && all_zero(file_buffer)) { 2527 file_buffer->c_byte = 0; 2528 seq_queue_put(to_main, file_buffer); 2529 } else { 2530 write_buffer->c_byte = mangle2(stream, 2531 write_buffer->data, file_buffer->data, 2532 file_buffer->size, block_size, 2533 file_buffer->noD, 1); 2534 write_buffer->sequence = file_buffer->sequence; 2535 write_buffer->file_size = file_buffer->file_size; 2536 write_buffer->block = file_buffer->block; 2537 write_buffer->size = SQUASHFS_COMPRESSED_SIZE_BLOCK 2538 (write_buffer->c_byte); 2539 write_buffer->fragment = FALSE; 2540 write_buffer->error = FALSE; 2541 cache_block_put(file_buffer); 2542 seq_queue_put(to_main, write_buffer); 2543 write_buffer = cache_get_nohash(bwriter_buffer); 2544 } 2545 } 2546 } 2547 2548 2549 void *frag_deflator(void *arg) 2550 { 2551 void *stream = NULL; 2552 int res; 2553 2554 res = compressor_init(comp, &stream, block_size, 1); 2555 if(res) 2556 BAD_ERROR("frag_deflator:: compressor_init failed\n"); 2557 2558 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 2559 2560 while(1) { 2561 int c_byte, compressed_size; 2562 struct file_buffer *file_buffer = queue_get(to_frag); 2563 struct file_buffer *write_buffer = 2564 cache_get(fwriter_buffer, file_buffer->block); 2565 2566 c_byte = mangle2(stream, write_buffer->data, file_buffer->data, 2567 file_buffer->size, block_size, noF, 1); 2568 compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); 2569 write_buffer->size = compressed_size; 2570 pthread_mutex_lock(&fragment_mutex); 2571 if(fragments_locked == FALSE) { 2572 fragment_table[file_buffer->block].size = c_byte; 2573 fragment_table[file_buffer->block].start_block = bytes; 2574 write_buffer->block = bytes; 2575 bytes += compressed_size; 2576 fragments_outstanding --; 2577 queue_put(to_writer, write_buffer); 2578 pthread_mutex_unlock(&fragment_mutex); 2579 TRACE("Writing fragment %lld, uncompressed size %d, " 2580 "compressed size %d\n", file_buffer->block, 2581 file_buffer->size, compressed_size); 2582 } else { 2583 add_pending_fragment(write_buffer, c_byte, 2584 file_buffer->block); 2585 pthread_mutex_unlock(&fragment_mutex); 2586 } 2587 cache_block_put(file_buffer); 2588 } 2589 2590 pthread_cleanup_pop(0); 2591 } 2592 2593 2594 struct file_buffer *get_file_buffer() 2595 { 2596 struct file_buffer *file_buffer = seq_queue_get(to_main); 2597 2598 return file_buffer; 2599 } 2600 2601 2602 void write_file_empty(squashfs_inode *inode, struct dir_ent *dir_ent, 2603 struct file_buffer *file_buffer, int *duplicate_file) 2604 { 2605 file_count ++; 2606 *duplicate_file = FALSE; 2607 cache_block_put(file_buffer); 2608 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, 0, 0, 0, 2609 NULL, &empty_fragment, NULL, 0); 2610 } 2611 2612 2613 void write_file_frag(squashfs_inode *inode, struct dir_ent *dir_ent, 2614 struct file_buffer *file_buffer, int *duplicate_file) 2615 { 2616 int size = file_buffer->file_size; 2617 struct fragment *fragment; 2618 unsigned short checksum = file_buffer->checksum; 2619 char dont_put; 2620 2621 fragment = frag_duplicate(file_buffer, &dont_put); 2622 *duplicate_file = !fragment; 2623 if(!fragment) { 2624 fragment = get_and_fill_fragment(file_buffer, dir_ent); 2625 if(duplicate_checking) 2626 add_non_dup(size, 0, NULL, 0, fragment, 0, checksum, 2627 TRUE, TRUE); 2628 } 2629 2630 if(dont_put) 2631 free(file_buffer); 2632 else 2633 cache_block_put(file_buffer); 2634 2635 total_bytes += size; 2636 file_count ++; 2637 2638 inc_progress_bar(); 2639 2640 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, size, 0, 2641 0, NULL, fragment, NULL, 0); 2642 2643 if(!duplicate_checking) 2644 free_fragment(fragment); 2645 } 2646 2647 2648 int write_file_process(squashfs_inode *inode, struct dir_ent *dir_ent, 2649 struct file_buffer *read_buffer, int *duplicate_file) 2650 { 2651 long long read_size, file_bytes, start; 2652 struct fragment *fragment; 2653 unsigned int *block_list = NULL; 2654 int block = 0, status; 2655 long long sparse = 0; 2656 struct file_buffer *fragment_buffer = NULL; 2657 2658 *duplicate_file = FALSE; 2659 2660 lock_fragments(); 2661 2662 file_bytes = 0; 2663 start = bytes; 2664 while (1) { 2665 read_size = read_buffer->file_size; 2666 if(read_buffer->fragment) 2667 fragment_buffer = read_buffer; 2668 else { 2669 block_list = realloc(block_list, (block + 1) * 2670 sizeof(unsigned int)); 2671 if(block_list == NULL) 2672 MEM_ERROR(); 2673 block_list[block ++] = read_buffer->c_byte; 2674 if(read_buffer->c_byte) { 2675 read_buffer->block = bytes; 2676 bytes += read_buffer->size; 2677 cache_hash(read_buffer, read_buffer->block); 2678 file_bytes += read_buffer->size; 2679 queue_put(to_writer, read_buffer); 2680 } else { 2681 sparse += read_buffer->size; 2682 cache_block_put(read_buffer); 2683 } 2684 } 2685 inc_progress_bar(); 2686 2687 if(read_size != -1) 2688 break; 2689 2690 read_buffer = get_file_buffer(); 2691 if(read_buffer->error) 2692 goto read_err; 2693 } 2694 2695 unlock_fragments(); 2696 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2697 2698 if(duplicate_checking) 2699 add_non_dup(read_size, file_bytes, block_list, start, fragment, 2700 0, fragment_buffer ? fragment_buffer->checksum : 0, 2701 FALSE, TRUE); 2702 cache_block_put(fragment_buffer); 2703 file_count ++; 2704 total_bytes += read_size; 2705 2706 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, 2707 block, block_list, fragment, NULL, sparse); 2708 2709 if(duplicate_checking == FALSE) { 2710 free(block_list); 2711 free_fragment(fragment); 2712 } 2713 2714 return 0; 2715 2716 read_err: 2717 dec_progress_bar(block); 2718 status = read_buffer->error; 2719 bytes = start; 2720 if(!block_device) { 2721 int res; 2722 2723 queue_put(to_writer, NULL); 2724 if(queue_get(from_writer) != 0) 2725 EXIT_MKSQUASHFS(); 2726 res = ftruncate(fd, bytes); 2727 if(res != 0) 2728 BAD_ERROR("Failed to truncate dest file because %s\n", 2729 strerror(errno)); 2730 } 2731 unlock_fragments(); 2732 free(block_list); 2733 cache_block_put(read_buffer); 2734 return status; 2735 } 2736 2737 2738 int write_file_blocks_dup(squashfs_inode *inode, struct dir_ent *dir_ent, 2739 struct file_buffer *read_buffer, int *duplicate_file) 2740 { 2741 int block, thresh; 2742 long long read_size = read_buffer->file_size; 2743 long long file_bytes, dup_start, start; 2744 struct fragment *fragment; 2745 struct file_info *dupl_ptr; 2746 int blocks = (read_size + block_size - 1) >> block_log; 2747 unsigned int *block_list, *block_listp; 2748 struct file_buffer **buffer_list; 2749 int status; 2750 long long sparse = 0; 2751 struct file_buffer *fragment_buffer = NULL; 2752 2753 block_list = malloc(blocks * sizeof(unsigned int)); 2754 if(block_list == NULL) 2755 MEM_ERROR(); 2756 block_listp = block_list; 2757 2758 buffer_list = malloc(blocks * sizeof(struct file_buffer *)); 2759 if(buffer_list == NULL) 2760 MEM_ERROR(); 2761 2762 lock_fragments(); 2763 2764 file_bytes = 0; 2765 start = dup_start = bytes; 2766 thresh = blocks > bwriter_size ? blocks - bwriter_size : 0; 2767 2768 for(block = 0; block < blocks;) { 2769 if(read_buffer->fragment) { 2770 block_list[block] = 0; 2771 buffer_list[block] = NULL; 2772 fragment_buffer = read_buffer; 2773 blocks = read_size >> block_log; 2774 } else { 2775 block_list[block] = read_buffer->c_byte; 2776 2777 if(read_buffer->c_byte) { 2778 read_buffer->block = bytes; 2779 bytes += read_buffer->size; 2780 file_bytes += read_buffer->size; 2781 cache_hash(read_buffer, read_buffer->block); 2782 if(block < thresh) { 2783 buffer_list[block] = NULL; 2784 queue_put(to_writer, read_buffer); 2785 } else 2786 buffer_list[block] = read_buffer; 2787 } else { 2788 buffer_list[block] = NULL; 2789 sparse += read_buffer->size; 2790 cache_block_put(read_buffer); 2791 } 2792 } 2793 inc_progress_bar(); 2794 2795 if(++block < blocks) { 2796 read_buffer = get_file_buffer(); 2797 if(read_buffer->error) 2798 goto read_err; 2799 } 2800 } 2801 2802 dupl_ptr = duplicate(read_size, file_bytes, &block_listp, &dup_start, 2803 &fragment, fragment_buffer, blocks, 0, FALSE); 2804 2805 if(dupl_ptr) { 2806 *duplicate_file = FALSE; 2807 for(block = thresh; block < blocks; block ++) 2808 if(buffer_list[block]) 2809 queue_put(to_writer, buffer_list[block]); 2810 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2811 dupl_ptr->fragment = fragment; 2812 } else { 2813 *duplicate_file = TRUE; 2814 for(block = thresh; block < blocks; block ++) 2815 cache_block_put(buffer_list[block]); 2816 bytes = start; 2817 if(thresh && !block_device) { 2818 int res; 2819 2820 queue_put(to_writer, NULL); 2821 if(queue_get(from_writer) != 0) 2822 EXIT_MKSQUASHFS(); 2823 res = ftruncate(fd, bytes); 2824 if(res != 0) 2825 BAD_ERROR("Failed to truncate dest file because" 2826 " %s\n", strerror(errno)); 2827 } 2828 } 2829 2830 unlock_fragments(); 2831 cache_block_put(fragment_buffer); 2832 free(buffer_list); 2833 file_count ++; 2834 total_bytes += read_size; 2835 2836 /* 2837 * sparse count is needed to ensure squashfs correctly reports a 2838 * a smaller block count on stat calls to sparse files. This is 2839 * to ensure intelligent applications like cp correctly handle the 2840 * file as a sparse file. If the file in the original filesystem isn't 2841 * stored as a sparse file then still store it sparsely in squashfs, but 2842 * report it as non-sparse on stat calls to preserve semantics 2843 */ 2844 if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) 2845 sparse = 0; 2846 2847 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, 2848 dup_start, blocks, block_listp, fragment, NULL, sparse); 2849 2850 if(*duplicate_file == TRUE) 2851 free(block_list); 2852 2853 return 0; 2854 2855 read_err: 2856 dec_progress_bar(block); 2857 status = read_buffer->error; 2858 bytes = start; 2859 if(thresh && !block_device) { 2860 int res; 2861 2862 queue_put(to_writer, NULL); 2863 if(queue_get(from_writer) != 0) 2864 EXIT_MKSQUASHFS(); 2865 res = ftruncate(fd, bytes); 2866 if(res != 0) 2867 BAD_ERROR("Failed to truncate dest file because %s\n", 2868 strerror(errno)); 2869 } 2870 unlock_fragments(); 2871 for(blocks = thresh; blocks < block; blocks ++) 2872 cache_block_put(buffer_list[blocks]); 2873 free(buffer_list); 2874 free(block_list); 2875 cache_block_put(read_buffer); 2876 return status; 2877 } 2878 2879 2880 int write_file_blocks(squashfs_inode *inode, struct dir_ent *dir_ent, 2881 struct file_buffer *read_buffer, int *dup) 2882 { 2883 long long read_size = read_buffer->file_size; 2884 long long file_bytes, start; 2885 struct fragment *fragment; 2886 unsigned int *block_list; 2887 int block, status; 2888 int blocks = (read_size + block_size - 1) >> block_log; 2889 long long sparse = 0; 2890 struct file_buffer *fragment_buffer = NULL; 2891 2892 if(pre_duplicate(read_size)) 2893 return write_file_blocks_dup(inode, dir_ent, read_buffer, dup); 2894 2895 *dup = FALSE; 2896 2897 block_list = malloc(blocks * sizeof(unsigned int)); 2898 if(block_list == NULL) 2899 MEM_ERROR(); 2900 2901 lock_fragments(); 2902 2903 file_bytes = 0; 2904 /* ANDROID CHANGES START*/ 2905 #ifdef ANDROID 2906 if (align_4k_blocks && bytes % 4096) { 2907 bytes += 4096 - (bytes % 4096); 2908 } 2909 #endif 2910 /* ANDROID CHANGES END */ 2911 start = bytes; 2912 for(block = 0; block < blocks;) { 2913 if(read_buffer->fragment) { 2914 block_list[block] = 0; 2915 fragment_buffer = read_buffer; 2916 blocks = read_size >> block_log; 2917 } else { 2918 block_list[block] = read_buffer->c_byte; 2919 if(read_buffer->c_byte) { 2920 read_buffer->block = bytes; 2921 bytes += read_buffer->size; 2922 cache_hash(read_buffer, read_buffer->block); 2923 file_bytes += read_buffer->size; 2924 queue_put(to_writer, read_buffer); 2925 } else { 2926 sparse += read_buffer->size; 2927 cache_block_put(read_buffer); 2928 } 2929 } 2930 inc_progress_bar(); 2931 2932 if(++block < blocks) { 2933 read_buffer = get_file_buffer(); 2934 if(read_buffer->error) 2935 goto read_err; 2936 } 2937 } 2938 2939 unlock_fragments(); 2940 fragment = get_and_fill_fragment(fragment_buffer, dir_ent); 2941 2942 if(duplicate_checking) 2943 add_non_dup(read_size, file_bytes, block_list, start, fragment, 2944 0, fragment_buffer ? fragment_buffer->checksum : 0, 2945 FALSE, TRUE); 2946 cache_block_put(fragment_buffer); 2947 file_count ++; 2948 total_bytes += read_size; 2949 2950 /* 2951 * sparse count is needed to ensure squashfs correctly reports a 2952 * a smaller block count on stat calls to sparse files. This is 2953 * to ensure intelligent applications like cp correctly handle the 2954 * file as a sparse file. If the file in the original filesystem isn't 2955 * stored as a sparse file then still store it sparsely in squashfs, but 2956 * report it as non-sparse on stat calls to preserve semantics 2957 */ 2958 if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) 2959 sparse = 0; 2960 2961 create_inode(inode, NULL, dir_ent, SQUASHFS_FILE_TYPE, read_size, start, 2962 blocks, block_list, fragment, NULL, sparse); 2963 2964 if(duplicate_checking == FALSE) { 2965 free(block_list); 2966 free_fragment(fragment); 2967 } 2968 2969 return 0; 2970 2971 read_err: 2972 dec_progress_bar(block); 2973 status = read_buffer->error; 2974 bytes = start; 2975 if(!block_device) { 2976 int res; 2977 2978 queue_put(to_writer, NULL); 2979 if(queue_get(from_writer) != 0) 2980 EXIT_MKSQUASHFS(); 2981 res = ftruncate(fd, bytes); 2982 if(res != 0) 2983 BAD_ERROR("Failed to truncate dest file because %s\n", 2984 strerror(errno)); 2985 } 2986 unlock_fragments(); 2987 free(block_list); 2988 cache_block_put(read_buffer); 2989 return status; 2990 } 2991 2992 2993 void write_file(squashfs_inode *inode, struct dir_ent *dir, int *dup) 2994 { 2995 int status; 2996 struct file_buffer *read_buffer; 2997 2998 again: 2999 read_buffer = get_file_buffer(); 3000 status = read_buffer->error; 3001 3002 if(status) 3003 cache_block_put(read_buffer); 3004 else if(read_buffer->file_size == -1) 3005 status = write_file_process(inode, dir, read_buffer, dup); 3006 else if(read_buffer->file_size == 0) 3007 write_file_empty(inode, dir, read_buffer, dup); 3008 else if(read_buffer->fragment && read_buffer->c_byte) 3009 write_file_frag(inode, dir, read_buffer, dup); 3010 else 3011 status = write_file_blocks(inode, dir, read_buffer, dup); 3012 3013 if(status == 2) { 3014 ERROR("File %s changed size while reading filesystem, " 3015 "attempting to re-read\n", pathname(dir)); 3016 goto again; 3017 } else if(status == 1) { 3018 ERROR_START("Failed to read file %s", pathname(dir)); 3019 ERROR_EXIT(", creating empty file\n"); 3020 write_file_empty(inode, dir, NULL, dup); 3021 } 3022 } 3023 3024 3025 #define BUFF_SIZE 512 3026 char *name; 3027 char *basename_r(); 3028 3029 char *getbase(char *pathname) 3030 { 3031 static char *b_buffer = NULL; 3032 static int b_size = BUFF_SIZE; 3033 char *result; 3034 3035 if(b_buffer == NULL) { 3036 b_buffer = malloc(b_size); 3037 if(b_buffer == NULL) 3038 MEM_ERROR(); 3039 } 3040 3041 while(1) { 3042 if(*pathname != '/') { 3043 result = getcwd(b_buffer, b_size); 3044 if(result == NULL && errno != ERANGE) 3045 BAD_ERROR("Getcwd failed in getbase\n"); 3046 3047 /* enough room for pathname + "/" + '\0' terminator? */ 3048 if(result && strlen(pathname) + 2 <= 3049 b_size - strlen(b_buffer)) { 3050 strcat(strcat(b_buffer, "/"), pathname); 3051 break; 3052 } 3053 } else if(strlen(pathname) < b_size) { 3054 strcpy(b_buffer, pathname); 3055 break; 3056 } 3057 3058 /* Buffer not large enough, realloc and try again */ 3059 b_buffer = realloc(b_buffer, b_size += BUFF_SIZE); 3060 if(b_buffer == NULL) 3061 MEM_ERROR(); 3062 } 3063 3064 name = b_buffer; 3065 if(((result = basename_r()) == NULL) || (strcmp(result, "..") == 0)) 3066 return NULL; 3067 else 3068 return result; 3069 } 3070 3071 3072 char *basename_r() 3073 { 3074 char *s; 3075 char *p; 3076 int n = 1; 3077 3078 for(;;) { 3079 s = name; 3080 if(*name == '\0') 3081 return NULL; 3082 if(*name != '/') { 3083 while(*name != '\0' && *name != '/') name++; 3084 n = name - s; 3085 } 3086 while(*name == '/') name++; 3087 if(strncmp(s, ".", n) == 0) 3088 continue; 3089 if((*name == '\0') || (strncmp(s, "..", n) == 0) || 3090 ((p = basename_r()) == NULL)) { 3091 s[n] = '\0'; 3092 return s; 3093 } 3094 if(strcmp(p, "..") == 0) 3095 continue; 3096 return p; 3097 } 3098 } 3099 3100 3101 struct inode_info *lookup_inode3(struct stat *buf, int pseudo, int id, 3102 char *symlink, int bytes) 3103 { 3104 int ino_hash = INODE_HASH(buf->st_dev, buf->st_ino); 3105 struct inode_info *inode; 3106 3107 /* 3108 * Look-up inode in hash table, if it already exists we have a 3109 * hard-link, so increment the nlink count and return it. 3110 * Don't do the look-up for directories because we don't hard-link 3111 * directories. 3112 */ 3113 if ((buf->st_mode & S_IFMT) != S_IFDIR) { 3114 for(inode = inode_info[ino_hash]; inode; inode = inode->next) { 3115 if(memcmp(buf, &inode->buf, sizeof(struct stat)) == 0) { 3116 inode->nlink ++; 3117 return inode; 3118 } 3119 } 3120 } 3121 3122 inode = malloc(sizeof(struct inode_info) + bytes); 3123 if(inode == NULL) 3124 MEM_ERROR(); 3125 3126 if(bytes) 3127 memcpy(&inode->symlink, symlink, bytes); 3128 memcpy(&inode->buf, buf, sizeof(struct stat)); 3129 inode->read = FALSE; 3130 inode->root_entry = FALSE; 3131 inode->pseudo_file = pseudo; 3132 inode->pseudo_id = id; 3133 inode->inode = SQUASHFS_INVALID_BLK; 3134 inode->nlink = 1; 3135 inode->inode_number = 0; 3136 3137 /* 3138 * Copy filesystem wide defaults into inode, these filesystem 3139 * wide defaults may be altered on an individual inode basis by 3140 * user specified actions 3141 * 3142 */ 3143 inode->no_fragments = no_fragments; 3144 inode->always_use_fragments = always_use_fragments; 3145 3146 /* ANDROID CHANGES START*/ 3147 #ifdef ANDROID 3148 /* Check the whitelist */ 3149 inode->noD = whitelisted(buf); 3150 #else 3151 inode->noD = noD; 3152 #endif 3153 /* ANDROID CHANGES END */ 3154 3155 inode->noF = noF; 3156 3157 inode->next = inode_info[ino_hash]; 3158 inode_info[ino_hash] = inode; 3159 3160 return inode; 3161 } 3162 3163 3164 static inline struct inode_info *lookup_inode2(struct stat *buf, int pseudo, int id) 3165 { 3166 return lookup_inode3(buf, pseudo, id, NULL, 0); 3167 } 3168 3169 3170 static inline struct inode_info *lookup_inode(struct stat *buf) 3171 { 3172 return lookup_inode2(buf, 0, 0); 3173 } 3174 3175 3176 static inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this) 3177 { 3178 if (inode->inode_number == 0) { 3179 inode->inode_number = use_this ? : inode_no ++; 3180 if((inode->buf.st_mode & S_IFMT) == S_IFREG) 3181 progress_bar_size((inode->buf.st_size + block_size - 1) 3182 >> block_log); 3183 } 3184 } 3185 3186 3187 static inline struct dir_ent *create_dir_entry(char *name, char *source_name, 3188 char *nonstandard_pathname, struct dir_info *dir) 3189 { 3190 struct dir_ent *dir_ent = malloc(sizeof(struct dir_ent)); 3191 if(dir_ent == NULL) 3192 MEM_ERROR(); 3193 3194 dir_ent->name = name; 3195 dir_ent->source_name = source_name; 3196 dir_ent->nonstandard_pathname = nonstandard_pathname; 3197 dir_ent->our_dir = dir; 3198 dir_ent->inode = NULL; 3199 dir_ent->next = NULL; 3200 /* ANDROID CHANGES START*/ 3201 #ifdef ANDROID 3202 dir_ent->capabilities = 0; 3203 #endif 3204 /* ANDROID CHANGES END */ 3205 3206 return dir_ent; 3207 } 3208 3209 3210 static inline void add_dir_entry(struct dir_ent *dir_ent, struct dir_info *sub_dir, 3211 struct inode_info *inode_info) 3212 { 3213 struct dir_info *dir = dir_ent->our_dir; 3214 3215 if(sub_dir) 3216 sub_dir->dir_ent = dir_ent; 3217 3218 /* ANDROID CHANGES START*/ 3219 #ifdef ANDROID 3220 if (android_config) { 3221 if (mount_point) { 3222 char *mounted_path; 3223 char *rel_path; 3224 3225 alloc_mounted_path(mount_point, subpathname(dir_ent), &mounted_path); 3226 rel_path = mounted_path; 3227 while (rel_path && *rel_path == '/') 3228 rel_path++; 3229 android_fs_config(fs_config_func, rel_path, &inode_info->buf, target_out_path, &dir_ent->capabilities); 3230 free(mounted_path); 3231 } else { 3232 android_fs_config(fs_config_func, pathname(dir_ent), &inode_info->buf, target_out_path, &dir_ent->capabilities); 3233 } 3234 } 3235 #endif 3236 /* ANDROID CHANGES END */ 3237 3238 dir_ent->inode = inode_info; 3239 dir_ent->dir = sub_dir; 3240 3241 dir_ent->next = dir->list; 3242 dir->list = dir_ent; 3243 dir->count++; 3244 } 3245 3246 static inline void add_dir_entry2(char *name, char *source_name, 3247 char *nonstandard_pathname, struct dir_info *sub_dir, 3248 struct inode_info *inode_info, struct dir_info *dir) 3249 { 3250 struct dir_ent *dir_ent = create_dir_entry(name, source_name, 3251 nonstandard_pathname, dir); 3252 3253 3254 add_dir_entry(dir_ent, sub_dir, inode_info); 3255 } 3256 3257 3258 static inline void free_dir_entry(struct dir_ent *dir_ent) 3259 { 3260 if(dir_ent->name) 3261 free(dir_ent->name); 3262 3263 if(dir_ent->source_name) 3264 free(dir_ent->source_name); 3265 3266 if(dir_ent->nonstandard_pathname) 3267 free(dir_ent->nonstandard_pathname); 3268 3269 /* if this entry has been associated with an inode, then we need 3270 * to update the inode nlink count. Orphaned inodes are harmless, and 3271 * is easier to leave them than go to the bother of deleting them */ 3272 if(dir_ent->inode && !dir_ent->inode->root_entry) 3273 dir_ent->inode->nlink --; 3274 3275 free(dir_ent); 3276 } 3277 3278 3279 static inline void add_excluded(struct dir_info *dir) 3280 { 3281 dir->excluded ++; 3282 } 3283 3284 3285 void dir_scan(squashfs_inode *inode, char *pathname, 3286 struct dir_ent *(_readdir)(struct dir_info *), int progress) 3287 { 3288 struct stat buf; 3289 struct dir_ent *dir_ent; 3290 /* ANDROID CHANGES START*/ 3291 #ifdef ANDROID 3292 uint64_t caps = 0; 3293 #endif 3294 /* ANDROID CHANGES END */ 3295 3296 root_dir = dir_scan1(pathname, "", paths, _readdir, 1); 3297 if(root_dir == NULL) 3298 return; 3299 3300 /* Create root directory dir_ent and associated inode, and connect 3301 * it to the root directory dir_info structure */ 3302 dir_ent = create_dir_entry("", NULL, pathname, 3303 scan1_opendir("", "", 0)); 3304 3305 if(pathname[0] == '\0') { 3306 /* 3307 * dummy top level directory, if multiple sources specified on 3308 * command line 3309 */ 3310 memset(&buf, 0, sizeof(buf)); 3311 buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR; 3312 buf.st_uid = getuid(); 3313 buf.st_gid = getgid(); 3314 buf.st_mtime = time(NULL); 3315 buf.st_dev = 0; 3316 buf.st_ino = 0; 3317 dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0); 3318 } else { 3319 if(lstat(pathname, &buf) == -1) 3320 /* source directory has disappeared? */ 3321 BAD_ERROR("Cannot stat source directory %s because %s\n", 3322 pathname, strerror(errno)); 3323 /* ANDROID CHANGES START*/ 3324 #ifdef ANDROID 3325 buf.st_mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH | S_IFDIR; // root mode 3326 buf.st_uid = 0; 3327 buf.st_gid = 0; 3328 buf.st_mtime = time(NULL); 3329 buf.st_dev = 0; 3330 buf.st_ino = 0; 3331 #endif 3332 /* ANDROID CHANGES END */ 3333 dir_ent->inode = lookup_inode(&buf); 3334 } 3335 3336 /* ANDROID CHANGES START*/ 3337 #ifdef ANDROID 3338 dir_ent->capabilities = caps; 3339 if (android_config) { 3340 android_fs_config(fs_config_func, "", &dir_ent->inode->buf, target_out_path, &dir_ent->capabilities); 3341 } 3342 #endif 3343 /* ANDROID CHANGES END */ 3344 3345 dir_ent->dir = root_dir; 3346 root_dir->dir_ent = dir_ent; 3347 3348 /* 3349 * Process most actions and any pseudo files 3350 */ 3351 if(actions() || get_pseudo()) 3352 dir_scan2(root_dir, get_pseudo()); 3353 3354 /* 3355 * Process move actions 3356 */ 3357 if(move_actions()) { 3358 dir_scan3(root_dir); 3359 do_move_actions(); 3360 } 3361 3362 /* 3363 * Process prune actions 3364 */ 3365 if(prune_actions()) 3366 dir_scan4(root_dir); 3367 3368 /* 3369 * Process empty actions 3370 */ 3371 if(empty_actions()) 3372 dir_scan5(root_dir); 3373 3374 /* 3375 * Sort directories and compute the inode numbers 3376 */ 3377 dir_scan6(root_dir); 3378 3379 alloc_inode_no(dir_ent->inode, root_inode_number); 3380 3381 eval_actions(root_dir, dir_ent); 3382 3383 if(sorted) 3384 generate_file_priorities(root_dir, 0, 3385 &root_dir->dir_ent->inode->buf); 3386 3387 if(appending) { 3388 sigset_t sigmask; 3389 3390 restore_thread = init_restore_thread(); 3391 sigemptyset(&sigmask); 3392 sigaddset(&sigmask, SIGINT); 3393 sigaddset(&sigmask, SIGTERM); 3394 sigaddset(&sigmask, SIGUSR1); 3395 if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1) 3396 BAD_ERROR("Failed to set signal mask\n"); 3397 write_destination(fd, SQUASHFS_START, 4, "\0\0\0\0"); 3398 } 3399 3400 queue_put(to_reader, root_dir); 3401 3402 set_progressbar_state(progress); 3403 3404 if(sorted) 3405 sort_files_and_write(root_dir); 3406 3407 dir_scan7(inode, root_dir); 3408 dir_ent->inode->inode = *inode; 3409 dir_ent->inode->type = SQUASHFS_DIR_TYPE; 3410 } 3411 3412 3413 /* 3414 * dir_scan1 routines... 3415 * These scan the source directories into memory for processing. 3416 * Exclude actions are processed here (in contrast to the other actions) 3417 * because they affect what is scanned. 3418 */ 3419 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth) 3420 { 3421 struct dir_info *dir; 3422 3423 dir = malloc(sizeof(struct dir_info)); 3424 if(dir == NULL) 3425 MEM_ERROR(); 3426 3427 if(pathname[0] != '\0') { 3428 dir->linuxdir = opendir(pathname); 3429 if(dir->linuxdir == NULL) { 3430 free(dir); 3431 return NULL; 3432 } 3433 } 3434 3435 dir->pathname = strdup(pathname); 3436 dir->subpath = strdup(subpath); 3437 dir->count = 0; 3438 dir->directory_count = 0; 3439 dir->dir_is_ldir = TRUE; 3440 dir->list = NULL; 3441 dir->depth = depth; 3442 dir->excluded = 0; 3443 3444 return dir; 3445 } 3446 3447 3448 struct dir_ent *scan1_encomp_readdir(struct dir_info *dir) 3449 { 3450 static int index = 0; 3451 3452 if(dir->count < old_root_entries) { 3453 int i; 3454 3455 for(i = 0; i < old_root_entries; i++) { 3456 if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) 3457 dir->directory_count ++; 3458 add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, 3459 &old_root_entry[i].inode, dir); 3460 } 3461 } 3462 3463 while(index < source) { 3464 char *basename = NULL; 3465 char *dir_name = getbase(source_path[index]); 3466 int pass = 1, res; 3467 3468 if(dir_name == NULL) { 3469 ERROR_START("Bad source directory %s", 3470 source_path[index]); 3471 ERROR_EXIT(" - skipping ...\n"); 3472 index ++; 3473 continue; 3474 } 3475 dir_name = strdup(dir_name); 3476 for(;;) { 3477 struct dir_ent *dir_ent = dir->list; 3478 3479 for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; 3480 dir_ent = dir_ent->next); 3481 if(dir_ent == NULL) 3482 break; 3483 ERROR("Source directory entry %s already used! - trying" 3484 " ", dir_name); 3485 if(pass == 1) 3486 basename = dir_name; 3487 else 3488 free(dir_name); 3489 res = asprintf(&dir_name, "%s_%d", basename, pass++); 3490 if(res == -1) 3491 BAD_ERROR("asprintf failed in " 3492 "scan1_encomp_readdir\n"); 3493 ERROR("%s\n", dir_name); 3494 } 3495 return create_dir_entry(dir_name, basename, 3496 strdup(source_path[index ++]), dir); 3497 } 3498 return NULL; 3499 } 3500 3501 3502 struct dir_ent *scan1_single_readdir(struct dir_info *dir) 3503 { 3504 struct dirent *d_name; 3505 int i; 3506 3507 if(dir->count < old_root_entries) { 3508 for(i = 0; i < old_root_entries; i++) { 3509 if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) 3510 dir->directory_count ++; 3511 add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, 3512 &old_root_entry[i].inode, dir); 3513 } 3514 } 3515 3516 if((d_name = readdir(dir->linuxdir)) != NULL) { 3517 char *basename = NULL; 3518 char *dir_name = strdup(d_name->d_name); 3519 int pass = 1, res; 3520 3521 for(;;) { 3522 struct dir_ent *dir_ent = dir->list; 3523 3524 for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; 3525 dir_ent = dir_ent->next); 3526 if(dir_ent == NULL) 3527 break; 3528 ERROR("Source directory entry %s already used! - trying" 3529 " ", dir_name); 3530 if (pass == 1) 3531 basename = dir_name; 3532 else 3533 free(dir_name); 3534 res = asprintf(&dir_name, "%s_%d", d_name->d_name, pass++); 3535 if(res == -1) 3536 BAD_ERROR("asprintf failed in " 3537 "scan1_single_readdir\n"); 3538 ERROR("%s\n", dir_name); 3539 } 3540 return create_dir_entry(dir_name, basename, NULL, dir); 3541 } 3542 3543 return NULL; 3544 } 3545 3546 3547 struct dir_ent *scan1_readdir(struct dir_info *dir) 3548 { 3549 struct dirent *d_name = readdir(dir->linuxdir); 3550 3551 return d_name ? 3552 create_dir_entry(strdup(d_name->d_name), NULL, NULL, dir) : 3553 NULL; 3554 } 3555 3556 3557 void scan1_freedir(struct dir_info *dir) 3558 { 3559 if(dir->pathname[0] != '\0') 3560 closedir(dir->linuxdir); 3561 } 3562 3563 3564 struct dir_info *dir_scan1(char *filename, char *subpath, 3565 struct pathnames *paths, 3566 struct dir_ent *(_readdir)(struct dir_info *), int depth) 3567 { 3568 struct dir_info *dir = scan1_opendir(filename, subpath, depth); 3569 struct dir_ent *dir_ent; 3570 3571 if(dir == NULL) { 3572 ERROR_START("Could not open %s", filename); 3573 ERROR_EXIT(", skipping...\n"); 3574 return NULL; 3575 } 3576 3577 while((dir_ent = _readdir(dir))) { 3578 struct dir_info *sub_dir; 3579 struct stat buf; 3580 struct pathnames *new = NULL; 3581 char *filename = pathname(dir_ent); 3582 char *subpath = NULL; 3583 char *dir_name = dir_ent->name; 3584 3585 if(strcmp(dir_name, ".") == 0 || strcmp(dir_name, "..") == 0) { 3586 free_dir_entry(dir_ent); 3587 continue; 3588 } 3589 3590 if(lstat(filename, &buf) == -1) { 3591 ERROR_START("Cannot stat dir/file %s because %s", 3592 filename, strerror(errno)); 3593 ERROR_EXIT(", ignoring\n"); 3594 free_dir_entry(dir_ent); 3595 continue; 3596 } 3597 3598 if((buf.st_mode & S_IFMT) != S_IFREG && 3599 (buf.st_mode & S_IFMT) != S_IFDIR && 3600 (buf.st_mode & S_IFMT) != S_IFLNK && 3601 (buf.st_mode & S_IFMT) != S_IFCHR && 3602 (buf.st_mode & S_IFMT) != S_IFBLK && 3603 (buf.st_mode & S_IFMT) != S_IFIFO && 3604 (buf.st_mode & S_IFMT) != S_IFSOCK) { 3605 ERROR_START("File %s has unrecognised filetype %d", 3606 filename, buf.st_mode & S_IFMT); 3607 ERROR_EXIT(", ignoring\n"); 3608 free_dir_entry(dir_ent); 3609 continue; 3610 } 3611 3612 if((old_exclude && old_excluded(filename, &buf)) || 3613 (!old_exclude && excluded(dir_name, paths, &new))) { 3614 add_excluded(dir); 3615 free_dir_entry(dir_ent); 3616 continue; 3617 } 3618 3619 if(exclude_actions()) { 3620 subpath = subpathname(dir_ent); 3621 3622 if(eval_exclude_actions(dir_name, filename, subpath, 3623 &buf, depth, dir_ent)) { 3624 add_excluded(dir); 3625 free_dir_entry(dir_ent); 3626 continue; 3627 } 3628 } 3629 3630 switch(buf.st_mode & S_IFMT) { 3631 case S_IFDIR: 3632 if(subpath == NULL) 3633 subpath = subpathname(dir_ent); 3634 3635 sub_dir = dir_scan1(filename, subpath, new, 3636 scan1_readdir, depth + 1); 3637 if(sub_dir) { 3638 dir->directory_count ++; 3639 add_dir_entry(dir_ent, sub_dir, 3640 lookup_inode(&buf)); 3641 } else 3642 free_dir_entry(dir_ent); 3643 break; 3644 case S_IFLNK: { 3645 int byte; 3646 static char buff[65536]; /* overflow safe */ 3647 3648 byte = readlink(filename, buff, 65536); 3649 if(byte == -1) { 3650 ERROR_START("Failed to read symlink %s", 3651 filename); 3652 ERROR_EXIT(", ignoring\n"); 3653 } else if(byte == 65536) { 3654 ERROR_START("Symlink %s is greater than 65536 " 3655 "bytes!", filename); 3656 ERROR_EXIT(", ignoring\n"); 3657 } else { 3658 /* readlink doesn't 0 terminate the returned 3659 * path */ 3660 buff[byte] = '\0'; 3661 add_dir_entry(dir_ent, NULL, lookup_inode3(&buf, 3662 0, 0, buff, byte + 1)); 3663 } 3664 break; 3665 } 3666 default: 3667 add_dir_entry(dir_ent, NULL, lookup_inode(&buf)); 3668 } 3669 3670 free(new); 3671 } 3672 3673 scan1_freedir(dir); 3674 3675 return dir; 3676 } 3677 3678 3679 /* 3680 * dir_scan2 routines... 3681 * This processes most actions and any pseudo files 3682 */ 3683 struct dir_ent *scan2_readdir(struct dir_info *dir, struct dir_ent *dir_ent) 3684 { 3685 if (dir_ent == NULL) 3686 dir_ent = dir->list; 3687 else 3688 dir_ent = dir_ent->next; 3689 3690 for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next); 3691 3692 return dir_ent; 3693 } 3694 3695 3696 struct dir_ent *scan2_lookup(struct dir_info *dir, char *name) 3697 { 3698 struct dir_ent *dir_ent = dir->list; 3699 3700 for(; dir_ent && strcmp(dir_ent->name, name) != 0; 3701 dir_ent = dir_ent->next); 3702 3703 return dir_ent; 3704 } 3705 3706 3707 void dir_scan2(struct dir_info *dir, struct pseudo *pseudo) 3708 { 3709 struct dir_ent *dir_ent = NULL; 3710 struct pseudo_entry *pseudo_ent; 3711 struct stat buf; 3712 static int pseudo_ino = 1; 3713 3714 while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) { 3715 struct inode_info *inode_info = dir_ent->inode; 3716 struct stat *buf = &inode_info->buf; 3717 char *name = dir_ent->name; 3718 3719 eval_actions(root_dir, dir_ent); 3720 3721 if((buf->st_mode & S_IFMT) == S_IFDIR) 3722 dir_scan2(dir_ent->dir, pseudo_subdir(name, pseudo)); 3723 } 3724 3725 while((pseudo_ent = pseudo_readdir(pseudo)) != NULL) { 3726 dir_ent = scan2_lookup(dir, pseudo_ent->name); 3727 if(pseudo_ent->dev->type == 'm') { 3728 struct stat *buf; 3729 if(dir_ent == NULL) { 3730 ERROR_START("Pseudo modify file \"%s\" does " 3731 "not exist in source filesystem.", 3732 pseudo_ent->pathname); 3733 ERROR_EXIT(" Ignoring.\n"); 3734 continue; 3735 } 3736 if(dir_ent->inode->root_entry) { 3737 ERROR_START("Pseudo modify file \"%s\" is a " 3738 "pre-existing file in the filesystem " 3739 "being appended to. It cannot be "\ 3740 "modified.", pseudo_ent->pathname); 3741 ERROR_EXIT(" Ignoring.\n"); 3742 continue; 3743 } 3744 buf = &dir_ent->inode->buf; 3745 buf->st_mode = (buf->st_mode & S_IFMT) | 3746 pseudo_ent->dev->mode; 3747 buf->st_uid = pseudo_ent->dev->uid; 3748 buf->st_gid = pseudo_ent->dev->gid; 3749 continue; 3750 } 3751 3752 if(dir_ent) { 3753 if(dir_ent->inode->root_entry) { 3754 ERROR_START("Pseudo file \"%s\" is a " 3755 "pre-existing file in the filesystem " 3756 "being appended to.", 3757 pseudo_ent->pathname); 3758 ERROR_EXIT(" Ignoring.\n"); 3759 } else { 3760 ERROR_START("Pseudo file \"%s\" exists in " 3761 "source filesystem \"%s\".", 3762 pseudo_ent->pathname, 3763 pathname(dir_ent)); 3764 ERROR_EXIT("\nIgnoring, exclude it (-e/-ef) to " 3765 "override.\n"); 3766 } 3767 continue; 3768 } 3769 3770 memset(&buf, 0, sizeof(buf)); 3771 buf.st_mode = pseudo_ent->dev->mode; 3772 buf.st_uid = pseudo_ent->dev->uid; 3773 buf.st_gid = pseudo_ent->dev->gid; 3774 buf.st_rdev = makedev(pseudo_ent->dev->major, 3775 pseudo_ent->dev->minor); 3776 buf.st_mtime = time(NULL); 3777 buf.st_ino = pseudo_ino ++; 3778 3779 if(pseudo_ent->dev->type == 'd') { 3780 struct dir_ent *dir_ent = 3781 create_dir_entry(pseudo_ent->name, NULL, 3782 pseudo_ent->pathname, dir); 3783 char *subpath = strdup(subpathname(dir_ent)); 3784 struct dir_info *sub_dir = scan1_opendir("", subpath, 3785 dir->depth + 1); 3786 if(sub_dir == NULL) { 3787 ERROR_START("Could not create pseudo directory " 3788 "\"%s\"", pseudo_ent->pathname); 3789 ERROR_EXIT(", skipping...\n"); 3790 free(subpath); 3791 pseudo_ino --; 3792 continue; 3793 } 3794 dir_scan2(sub_dir, pseudo_ent->pseudo); 3795 dir->directory_count ++; 3796 add_dir_entry(dir_ent, sub_dir, 3797 lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0)); 3798 } else if(pseudo_ent->dev->type == 'f') { 3799 add_dir_entry2(pseudo_ent->name, NULL, 3800 pseudo_ent->pathname, NULL, 3801 lookup_inode2(&buf, PSEUDO_FILE_PROCESS, 3802 pseudo_ent->dev->pseudo_id), dir); 3803 } else { 3804 add_dir_entry2(pseudo_ent->name, NULL, 3805 pseudo_ent->pathname, NULL, 3806 lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0), dir); 3807 } 3808 } 3809 } 3810 3811 3812 /* 3813 * dir_scan3 routines... 3814 * This processes the move action 3815 */ 3816 void dir_scan3(struct dir_info *dir) 3817 { 3818 struct dir_ent *dir_ent = NULL; 3819 3820 while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) { 3821 3822 eval_move_actions(root_dir, dir_ent); 3823 3824 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3825 dir_scan3(dir_ent->dir); 3826 } 3827 } 3828 3829 3830 /* 3831 * dir_scan4 routines... 3832 * This processes the prune action. This action is designed to do fine 3833 * grained tuning of the in-core directory structure after the exclude, 3834 * move and pseudo actions have been performed. This allows complex 3835 * tests to be performed which are impossible at exclude time (i.e. 3836 * tests which rely on the in-core directory structure) 3837 */ 3838 void free_dir(struct dir_info *dir) 3839 { 3840 struct dir_ent *dir_ent = dir->list; 3841 3842 while(dir_ent) { 3843 struct dir_ent *tmp = dir_ent; 3844 3845 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3846 free_dir(dir_ent->dir); 3847 3848 dir_ent = dir_ent->next; 3849 free_dir_entry(tmp); 3850 } 3851 3852 free(dir->pathname); 3853 free(dir->subpath); 3854 free(dir); 3855 } 3856 3857 3858 void dir_scan4(struct dir_info *dir) 3859 { 3860 struct dir_ent *dir_ent = dir->list, *prev = NULL; 3861 3862 while(dir_ent) { 3863 if(dir_ent->inode->root_entry) { 3864 prev = dir_ent; 3865 dir_ent = dir_ent->next; 3866 continue; 3867 } 3868 3869 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 3870 dir_scan4(dir_ent->dir); 3871 3872 if(eval_prune_actions(root_dir, dir_ent)) { 3873 struct dir_ent *tmp = dir_ent; 3874 3875 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { 3876 free_dir(dir_ent->dir); 3877 dir->directory_count --; 3878 } 3879 3880 dir->count --; 3881 3882 /* remove dir_ent from list */ 3883 dir_ent = dir_ent->next; 3884 if(prev) 3885 prev->next = dir_ent; 3886 else 3887 dir->list = dir_ent; 3888 3889 /* free it */ 3890 free_dir_entry(tmp); 3891 3892 add_excluded(dir); 3893 continue; 3894 } 3895 3896 prev = dir_ent; 3897 dir_ent = dir_ent->next; 3898 } 3899 } 3900 3901 3902 /* 3903 * dir_scan5 routines... 3904 * This processes the empty action. This action has to be processed after 3905 * all other actions because the previous exclude and move actions and the 3906 * pseudo actions affect whether a directory is empty 3907 */ 3908 void dir_scan5(struct dir_info *dir) 3909 { 3910 struct dir_ent *dir_ent = dir->list, *prev = NULL; 3911 3912 while(dir_ent) { 3913 if(dir_ent->inode->root_entry) { 3914 prev = dir_ent; 3915 dir_ent = dir_ent->next; 3916 continue; 3917 } 3918 3919 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { 3920 dir_scan5(dir_ent->dir); 3921 3922 if(eval_empty_actions(root_dir, dir_ent)) { 3923 struct dir_ent *tmp = dir_ent; 3924 3925 /* 3926 * delete sub-directory, this is by definition 3927 * empty 3928 */ 3929 free(dir_ent->dir->pathname); 3930 free(dir_ent->dir->subpath); 3931 free(dir_ent->dir); 3932 3933 /* remove dir_ent from list */ 3934 dir_ent = dir_ent->next; 3935 if(prev) 3936 prev->next = dir_ent; 3937 else 3938 dir->list = dir_ent; 3939 3940 /* free it */ 3941 free_dir_entry(tmp); 3942 3943 /* update counts */ 3944 dir->directory_count --; 3945 dir->count --; 3946 add_excluded(dir); 3947 continue; 3948 } 3949 } 3950 3951 prev = dir_ent; 3952 dir_ent = dir_ent->next; 3953 } 3954 } 3955 3956 3957 /* 3958 * dir_scan6 routines... 3959 * This sorts every directory and computes the inode numbers 3960 */ 3961 3962 /* 3963 * Bottom up linked list merge sort. 3964 * 3965 * Qsort and other O(n log n) algorithms work well with arrays but not 3966 * linked lists. Merge sort another O(n log n) sort algorithm on the other hand 3967 * is not ideal for arrays (as it needs an additonal n storage locations 3968 * as sorting is not done in place), but it is ideal for linked lists because 3969 * it doesn't require any extra storage, 3970 */ 3971 void sort_directory(struct dir_info *dir) 3972 { 3973 struct dir_ent *cur, *l1, *l2, *next; 3974 int len1, len2, stride = 1; 3975 3976 if(dir->list == NULL || dir->count < 2) 3977 return; 3978 3979 /* 3980 * We can consider our linked-list to be made up of stride length 3981 * sublists. Eacn iteration around this loop merges adjacent 3982 * stride length sublists into larger 2*stride sublists. We stop 3983 * when stride becomes equal to the entire list. 3984 * 3985 * Initially stride = 1 (by definition a sublist of 1 is sorted), and 3986 * these 1 element sublists are merged into 2 element sublists, which 3987 * are then merged into 4 element sublists and so on. 3988 */ 3989 do { 3990 l2 = dir->list; /* head of current linked list */ 3991 cur = NULL; /* empty output list */ 3992 3993 /* 3994 * Iterate through the linked list, merging adjacent sublists. 3995 * On each interation l2 points to the next sublist pair to be 3996 * merged (if there's only one sublist left this is simply added 3997 * to the output list) 3998 */ 3999 while(l2) { 4000 l1 = l2; 4001 for(len1 = 0; l2 && len1 < stride; len1 ++, l2 = l2->next); 4002 len2 = stride; 4003 4004 /* 4005 * l1 points to first sublist. 4006 * l2 points to second sublist. 4007 * Merge them onto the output list 4008 */ 4009 while(len1 && l2 && len2) { 4010 if(strcmp(l1->name, l2->name) <= 0) { 4011 next = l1; 4012 l1 = l1->next; 4013 len1 --; 4014 } else { 4015 next = l2; 4016 l2 = l2->next; 4017 len2 --; 4018 } 4019 4020 if(cur) { 4021 cur->next = next; 4022 cur = next; 4023 } else 4024 dir->list = cur = next; 4025 } 4026 /* 4027 * One sublist is now empty, copy the other one onto the 4028 * output list 4029 */ 4030 for(; len1; len1 --, l1 = l1->next) { 4031 if(cur) { 4032 cur->next = l1; 4033 cur = l1; 4034 } else 4035 dir->list = cur = l1; 4036 } 4037 for(; l2 && len2; len2 --, l2 = l2->next) { 4038 if(cur) { 4039 cur->next = l2; 4040 cur = l2; 4041 } else 4042 dir->list = cur = l2; 4043 } 4044 } 4045 cur->next = NULL; 4046 stride = stride << 1; 4047 } while(stride < dir->count); 4048 } 4049 4050 4051 void dir_scan6(struct dir_info *dir) 4052 { 4053 struct dir_ent *dir_ent; 4054 unsigned int byte_count = 0; 4055 4056 sort_directory(dir); 4057 4058 for(dir_ent = dir->list; dir_ent; dir_ent = dir_ent->next) { 4059 byte_count += strlen(dir_ent->name) + 4060 sizeof(struct squashfs_dir_entry); 4061 4062 if(dir_ent->inode->root_entry) 4063 continue; 4064 4065 alloc_inode_no(dir_ent->inode, 0); 4066 4067 if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) 4068 dir_scan6(dir_ent->dir); 4069 } 4070 4071 if((dir->count < 257 && byte_count < SQUASHFS_METADATA_SIZE)) 4072 dir->dir_is_ldir = FALSE; 4073 } 4074 4075 4076 /* 4077 * dir_scan6 routines... 4078 * This generates the filesystem metadata and writes it out to the destination 4079 */ 4080 void scan7_init_dir(struct directory *dir) 4081 { 4082 dir->buff = malloc(SQUASHFS_METADATA_SIZE); 4083 if(dir->buff == NULL) 4084 MEM_ERROR(); 4085 4086 dir->size = SQUASHFS_METADATA_SIZE; 4087 dir->p = dir->index_count_p = dir->buff; 4088 dir->entry_count = 256; 4089 dir->entry_count_p = NULL; 4090 dir->index = NULL; 4091 dir->i_count = dir->i_size = 0; 4092 } 4093 4094 4095 struct dir_ent *scan7_readdir(struct directory *dir, struct dir_info *dir_info, 4096 struct dir_ent *dir_ent) 4097 { 4098 if (dir_ent == NULL) 4099 dir_ent = dir_info->list; 4100 else 4101 dir_ent = dir_ent->next; 4102 4103 for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next) 4104 add_dir(dir_ent->inode->inode, dir_ent->inode->inode_number, 4105 dir_ent->name, dir_ent->inode->type, dir); 4106 4107 return dir_ent; 4108 } 4109 4110 4111 void scan7_freedir(struct directory *dir) 4112 { 4113 if(dir->index) 4114 free(dir->index); 4115 free(dir->buff); 4116 } 4117 4118 4119 void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info) 4120 { 4121 int squashfs_type; 4122 int duplicate_file; 4123 struct directory dir; 4124 struct dir_ent *dir_ent = NULL; 4125 4126 scan7_init_dir(&dir); 4127 4128 while((dir_ent = scan7_readdir(&dir, dir_info, dir_ent)) != NULL) { 4129 struct stat *buf = &dir_ent->inode->buf; 4130 4131 update_info(dir_ent); 4132 4133 if(dir_ent->inode->inode == SQUASHFS_INVALID_BLK) { 4134 switch(buf->st_mode & S_IFMT) { 4135 case S_IFREG: 4136 squashfs_type = SQUASHFS_FILE_TYPE; 4137 write_file(inode, dir_ent, 4138 &duplicate_file); 4139 INFO("file %s, uncompressed size %lld " 4140 "bytes %s\n", 4141 subpathname(dir_ent), 4142 (long long) buf->st_size, 4143 duplicate_file ? "DUPLICATE" : 4144 ""); 4145 break; 4146 4147 case S_IFDIR: 4148 squashfs_type = SQUASHFS_DIR_TYPE; 4149 dir_scan7(inode, dir_ent->dir); 4150 break; 4151 4152 case S_IFLNK: 4153 squashfs_type = SQUASHFS_SYMLINK_TYPE; 4154 create_inode(inode, NULL, dir_ent, 4155 squashfs_type, 0, 0, 0, NULL, 4156 NULL, NULL, 0); 4157 INFO("symbolic link %s inode 0x%llx\n", 4158 subpathname(dir_ent), *inode); 4159 sym_count ++; 4160 break; 4161 4162 case S_IFCHR: 4163 squashfs_type = SQUASHFS_CHRDEV_TYPE; 4164 create_inode(inode, NULL, dir_ent, 4165 squashfs_type, 0, 0, 0, NULL, 4166 NULL, NULL, 0); 4167 INFO("character device %s inode 0x%llx" 4168 "\n", subpathname(dir_ent), 4169 *inode); 4170 dev_count ++; 4171 break; 4172 4173 case S_IFBLK: 4174 squashfs_type = SQUASHFS_BLKDEV_TYPE; 4175 create_inode(inode, NULL, dir_ent, 4176 squashfs_type, 0, 0, 0, NULL, 4177 NULL, NULL, 0); 4178 INFO("block device %s inode 0x%llx\n", 4179 subpathname(dir_ent), *inode); 4180 dev_count ++; 4181 break; 4182 4183 case S_IFIFO: 4184 squashfs_type = SQUASHFS_FIFO_TYPE; 4185 create_inode(inode, NULL, dir_ent, 4186 squashfs_type, 0, 0, 0, NULL, 4187 NULL, NULL, 0); 4188 INFO("fifo %s inode 0x%llx\n", 4189 subpathname(dir_ent), *inode); 4190 fifo_count ++; 4191 break; 4192 4193 case S_IFSOCK: 4194 squashfs_type = SQUASHFS_SOCKET_TYPE; 4195 create_inode(inode, NULL, dir_ent, 4196 squashfs_type, 0, 0, 0, NULL, 4197 NULL, NULL, 0); 4198 INFO("unix domain socket %s inode " 4199 "0x%llx\n", 4200 subpathname(dir_ent), *inode); 4201 sock_count ++; 4202 break; 4203 4204 default: 4205 BAD_ERROR("%s unrecognised file type, " 4206 "mode is %x\n", 4207 subpathname(dir_ent), 4208 buf->st_mode); 4209 } 4210 dir_ent->inode->inode = *inode; 4211 dir_ent->inode->type = squashfs_type; 4212 } else { 4213 *inode = dir_ent->inode->inode; 4214 squashfs_type = dir_ent->inode->type; 4215 switch(squashfs_type) { 4216 case SQUASHFS_FILE_TYPE: 4217 if(!sorted) 4218 INFO("file %s, uncompressed " 4219 "size %lld bytes LINK" 4220 "\n", 4221 subpathname(dir_ent), 4222 (long long) 4223 buf->st_size); 4224 break; 4225 case SQUASHFS_SYMLINK_TYPE: 4226 INFO("symbolic link %s inode 0x%llx " 4227 "LINK\n", subpathname(dir_ent), 4228 *inode); 4229 break; 4230 case SQUASHFS_CHRDEV_TYPE: 4231 INFO("character device %s inode 0x%llx " 4232 "LINK\n", subpathname(dir_ent), 4233 *inode); 4234 break; 4235 case SQUASHFS_BLKDEV_TYPE: 4236 INFO("block device %s inode 0x%llx " 4237 "LINK\n", subpathname(dir_ent), 4238 *inode); 4239 break; 4240 case SQUASHFS_FIFO_TYPE: 4241 INFO("fifo %s inode 0x%llx LINK\n", 4242 subpathname(dir_ent), *inode); 4243 break; 4244 case SQUASHFS_SOCKET_TYPE: 4245 INFO("unix domain socket %s inode " 4246 "0x%llx LINK\n", 4247 subpathname(dir_ent), *inode); 4248 break; 4249 } 4250 } 4251 4252 add_dir(*inode, get_inode_no(dir_ent->inode), dir_ent->name, 4253 squashfs_type, &dir); 4254 } 4255 4256 write_dir(inode, dir_info, &dir); 4257 INFO("directory %s inode 0x%llx\n", subpathname(dir_info->dir_ent), 4258 *inode); 4259 4260 scan7_freedir(&dir); 4261 } 4262 4263 4264 unsigned int slog(unsigned int block) 4265 { 4266 int i; 4267 4268 for(i = 12; i <= 20; i++) 4269 if(block == (1 << i)) 4270 return i; 4271 return 0; 4272 } 4273 4274 4275 int old_excluded(char *filename, struct stat *buf) 4276 { 4277 int i; 4278 4279 for(i = 0; i < exclude; i++) 4280 if((exclude_paths[i].st_dev == buf->st_dev) && 4281 (exclude_paths[i].st_ino == buf->st_ino)) 4282 return TRUE; 4283 return FALSE; 4284 } 4285 4286 4287 #define ADD_ENTRY(buf) \ 4288 if(exclude % EXCLUDE_SIZE == 0) { \ 4289 exclude_paths = realloc(exclude_paths, (exclude + EXCLUDE_SIZE) \ 4290 * sizeof(struct exclude_info)); \ 4291 if(exclude_paths == NULL) \ 4292 MEM_ERROR(); \ 4293 } \ 4294 exclude_paths[exclude].st_dev = buf.st_dev; \ 4295 exclude_paths[exclude++].st_ino = buf.st_ino; 4296 int old_add_exclude(char *path) 4297 { 4298 int i; 4299 char *filename; 4300 struct stat buf; 4301 4302 if(path[0] == '/' || strncmp(path, "./", 2) == 0 || 4303 strncmp(path, "../", 3) == 0) { 4304 if(lstat(path, &buf) == -1) { 4305 ERROR_START("Cannot stat exclude dir/file %s because " 4306 "%s", path, strerror(errno)); 4307 ERROR_EXIT(", ignoring\n"); 4308 return TRUE; 4309 } 4310 ADD_ENTRY(buf); 4311 return TRUE; 4312 } 4313 4314 for(i = 0; i < source; i++) { 4315 int res = asprintf(&filename, "%s/%s", source_path[i], path); 4316 if(res == -1) 4317 BAD_ERROR("asprintf failed in old_add_exclude\n"); 4318 if(lstat(filename, &buf) == -1) { 4319 if(!(errno == ENOENT || errno == ENOTDIR)) { 4320 ERROR_START("Cannot stat exclude dir/file %s " 4321 "because %s", filename, strerror(errno)); 4322 ERROR_EXIT(", ignoring\n"); 4323 } 4324 free(filename); 4325 continue; 4326 } 4327 free(filename); 4328 ADD_ENTRY(buf); 4329 } 4330 return TRUE; 4331 } 4332 4333 4334 void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, 4335 int type) 4336 { 4337 old_root_entry = realloc(old_root_entry, 4338 sizeof(struct old_root_entry_info) * (old_root_entries + 1)); 4339 if(old_root_entry == NULL) 4340 MEM_ERROR(); 4341 4342 old_root_entry[old_root_entries].name = strdup(name); 4343 old_root_entry[old_root_entries].inode.inode = inode; 4344 old_root_entry[old_root_entries].inode.inode_number = inode_number; 4345 old_root_entry[old_root_entries].inode.type = type; 4346 old_root_entry[old_root_entries++].inode.root_entry = TRUE; 4347 } 4348 4349 4350 void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, 4351 int freelst, char *destination_file) 4352 { 4353 int i; 4354 sigset_t sigmask, old_mask; 4355 int total_mem = readq; 4356 int reader_size; 4357 int fragment_size; 4358 int fwriter_size; 4359 /* 4360 * bwriter_size is global because it is needed in 4361 * write_file_blocks_dup() 4362 */ 4363 4364 /* 4365 * Never allow the total size of the queues to be larger than 4366 * physical memory 4367 * 4368 * When adding together the possibly user supplied values, make 4369 * sure they've not been deliberately contrived to overflow an int 4370 */ 4371 if(add_overflow(total_mem, fragq)) 4372 BAD_ERROR("Queue sizes rediculously too large\n"); 4373 total_mem += fragq; 4374 if(add_overflow(total_mem, bwriteq)) 4375 BAD_ERROR("Queue sizes rediculously too large\n"); 4376 total_mem += bwriteq; 4377 if(add_overflow(total_mem, fwriteq)) 4378 BAD_ERROR("Queue sizes rediculously too large\n"); 4379 total_mem += fwriteq; 4380 4381 check_usable_phys_mem(total_mem); 4382 4383 /* 4384 * convert from queue size in Mbytes to queue size in 4385 * blocks. 4386 * 4387 * This isn't going to overflow an int unless there exists 4388 * systems with more than 8 Petabytes of RAM! 4389 */ 4390 reader_size = readq << (20 - block_log); 4391 fragment_size = fragq << (20 - block_log); 4392 bwriter_size = bwriteq << (20 - block_log); 4393 fwriter_size = fwriteq << (20 - block_log); 4394 4395 /* 4396 * setup signal handlers for the main thread, these cleanup 4397 * deleting the destination file, if appending the 4398 * handlers for SIGTERM and SIGINT will be replaced with handlers 4399 * allowing the user to press ^C twice to restore the existing 4400 * filesystem. 4401 * 4402 * SIGUSR1 is an internal signal, which is used by the sub-threads 4403 * to tell the main thread to terminate, deleting the destination file, 4404 * or if necessary restoring the filesystem on appending 4405 */ 4406 signal(SIGTERM, sighandler); 4407 signal(SIGINT, sighandler); 4408 signal(SIGUSR1, sighandler); 4409 4410 /* block SIGQUIT and SIGHUP, these are handled by the info thread */ 4411 sigemptyset(&sigmask); 4412 sigaddset(&sigmask, SIGQUIT); 4413 sigaddset(&sigmask, SIGHUP); 4414 sigaddset(&sigmask, SIGALRM); 4415 if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) == -1) 4416 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4417 4418 /* 4419 * temporarily block these signals, so the created sub-threads 4420 * will ignore them, ensuring the main thread handles them 4421 */ 4422 sigemptyset(&sigmask); 4423 sigaddset(&sigmask, SIGINT); 4424 sigaddset(&sigmask, SIGTERM); 4425 sigaddset(&sigmask, SIGUSR1); 4426 if(pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask) == -1) 4427 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4428 4429 if(processors == -1) { 4430 #ifndef linux 4431 int mib[2]; 4432 size_t len = sizeof(processors); 4433 4434 mib[0] = CTL_HW; 4435 #ifdef HW_AVAILCPU 4436 mib[1] = HW_AVAILCPU; 4437 #else 4438 mib[1] = HW_NCPU; 4439 #endif 4440 4441 if(sysctl(mib, 2, &processors, &len, NULL, 0) == -1) { 4442 ERROR_START("Failed to get number of available " 4443 "processors."); 4444 ERROR_EXIT(" Defaulting to 1\n"); 4445 processors = 1; 4446 } 4447 #else 4448 processors = sysconf(_SC_NPROCESSORS_ONLN); 4449 #endif 4450 } 4451 4452 if(multiply_overflow(processors, 3) || 4453 multiply_overflow(processors * 3, sizeof(pthread_t))) 4454 BAD_ERROR("Processors too large\n"); 4455 4456 deflator_thread = malloc(processors * 3 * sizeof(pthread_t)); 4457 if(deflator_thread == NULL) 4458 MEM_ERROR(); 4459 4460 frag_deflator_thread = &deflator_thread[processors]; 4461 frag_thread = &frag_deflator_thread[processors]; 4462 4463 to_reader = queue_init(1); 4464 to_deflate = queue_init(reader_size); 4465 to_process_frag = queue_init(reader_size); 4466 to_writer = queue_init(bwriter_size + fwriter_size); 4467 from_writer = queue_init(1); 4468 to_frag = queue_init(fragment_size); 4469 locked_fragment = queue_init(fragment_size); 4470 to_main = seq_queue_init(); 4471 reader_buffer = cache_init(block_size, reader_size, 0, 0); 4472 bwriter_buffer = cache_init(block_size, bwriter_size, 1, freelst); 4473 fwriter_buffer = cache_init(block_size, fwriter_size, 1, freelst); 4474 fragment_buffer = cache_init(block_size, fragment_size, 1, 0); 4475 reserve_cache = cache_init(block_size, processors + 1, 1, 0); 4476 pthread_create(&reader_thread, NULL, reader, NULL); 4477 pthread_create(&writer_thread, NULL, writer, NULL); 4478 init_progress_bar(); 4479 init_info(); 4480 4481 for(i = 0; i < processors; i++) { 4482 if(pthread_create(&deflator_thread[i], NULL, deflator, NULL)) 4483 BAD_ERROR("Failed to create thread\n"); 4484 if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator, 4485 NULL) != 0) 4486 BAD_ERROR("Failed to create thread\n"); 4487 if(pthread_create(&frag_thread[i], NULL, frag_thrd, 4488 (void *) destination_file) != 0) 4489 BAD_ERROR("Failed to create thread\n"); 4490 } 4491 4492 main_thread = pthread_self(); 4493 4494 printf("Parallel mksquashfs: Using %d processor%s\n", processors, 4495 processors == 1 ? "" : "s"); 4496 4497 /* Restore the signal mask for the main thread */ 4498 if(pthread_sigmask(SIG_SETMASK, &old_mask, NULL) == -1) 4499 BAD_ERROR("Failed to set signal mask in intialise_threads\n"); 4500 } 4501 4502 4503 long long write_inode_lookup_table() 4504 { 4505 int i, inode_number, lookup_bytes = SQUASHFS_LOOKUP_BYTES(inode_count); 4506 void *it; 4507 4508 if(inode_count == sinode_count) 4509 goto skip_inode_hash_table; 4510 4511 it = realloc(inode_lookup_table, lookup_bytes); 4512 if(it == NULL) 4513 MEM_ERROR(); 4514 inode_lookup_table = it; 4515 4516 for(i = 0; i < INODE_HASH_SIZE; i ++) { 4517 struct inode_info *inode; 4518 4519 for(inode = inode_info[i]; inode; inode = inode->next) { 4520 4521 inode_number = get_inode_no(inode); 4522 4523 /* The empty action will produce orphaned inode 4524 * entries in the inode_info[] table. These 4525 * entries because they are orphaned will not be 4526 * allocated an inode number in dir_scan5(), so 4527 * skip any entries with the default dummy inode 4528 * number of 0 */ 4529 if(inode_number == 0) 4530 continue; 4531 4532 SQUASHFS_SWAP_LONG_LONGS(&inode->inode, 4533 &inode_lookup_table[inode_number - 1], 1); 4534 4535 } 4536 } 4537 4538 skip_inode_hash_table: 4539 return generic_write_table(lookup_bytes, inode_lookup_table, 0, NULL, 4540 noI); 4541 } 4542 4543 4544 char *get_component(char *target, char **targname) 4545 { 4546 char *start; 4547 4548 while(*target == '/') 4549 target ++; 4550 4551 start = target; 4552 while(*target != '/' && *target != '\0') 4553 target ++; 4554 4555 *targname = strndup(start, target - start); 4556 4557 while(*target == '/') 4558 target ++; 4559 4560 return target; 4561 } 4562 4563 4564 void free_path(struct pathname *paths) 4565 { 4566 int i; 4567 4568 for(i = 0; i < paths->names; i++) { 4569 if(paths->name[i].paths) 4570 free_path(paths->name[i].paths); 4571 free(paths->name[i].name); 4572 if(paths->name[i].preg) { 4573 regfree(paths->name[i].preg); 4574 free(paths->name[i].preg); 4575 } 4576 } 4577 4578 free(paths); 4579 } 4580 4581 4582 struct pathname *add_path(struct pathname *paths, char *target, char *alltarget) 4583 { 4584 char *targname; 4585 int i, error; 4586 4587 target = get_component(target, &targname); 4588 4589 if(paths == NULL) { 4590 paths = malloc(sizeof(struct pathname)); 4591 if(paths == NULL) 4592 MEM_ERROR(); 4593 4594 paths->names = 0; 4595 paths->name = NULL; 4596 } 4597 4598 for(i = 0; i < paths->names; i++) 4599 if(strcmp(paths->name[i].name, targname) == 0) 4600 break; 4601 4602 if(i == paths->names) { 4603 /* allocate new name entry */ 4604 paths->names ++; 4605 paths->name = realloc(paths->name, (i + 1) * 4606 sizeof(struct path_entry)); 4607 if(paths->name == NULL) 4608 MEM_ERROR(); 4609 paths->name[i].name = targname; 4610 paths->name[i].paths = NULL; 4611 if(use_regex) { 4612 paths->name[i].preg = malloc(sizeof(regex_t)); 4613 if(paths->name[i].preg == NULL) 4614 MEM_ERROR(); 4615 error = regcomp(paths->name[i].preg, targname, 4616 REG_EXTENDED|REG_NOSUB); 4617 if(error) { 4618 char str[1024]; /* overflow safe */ 4619 4620 regerror(error, paths->name[i].preg, str, 1024); 4621 BAD_ERROR("invalid regex %s in export %s, " 4622 "because %s\n", targname, alltarget, 4623 str); 4624 } 4625 } else 4626 paths->name[i].preg = NULL; 4627 4628 if(target[0] == '\0') 4629 /* at leaf pathname component */ 4630 paths->name[i].paths = NULL; 4631 else 4632 /* recurse adding child components */ 4633 paths->name[i].paths = add_path(NULL, target, 4634 alltarget); 4635 } else { 4636 /* existing matching entry */ 4637 free(targname); 4638 4639 if(paths->name[i].paths == NULL) { 4640 /* No sub-directory which means this is the leaf 4641 * component of a pre-existing exclude which subsumes 4642 * the exclude currently being added, in which case stop 4643 * adding components */ 4644 } else if(target[0] == '\0') { 4645 /* at leaf pathname component and child components exist 4646 * from more specific excludes, delete as they're 4647 * subsumed by this exclude */ 4648 free_path(paths->name[i].paths); 4649 paths->name[i].paths = NULL; 4650 } else 4651 /* recurse adding child components */ 4652 add_path(paths->name[i].paths, target, alltarget); 4653 } 4654 4655 return paths; 4656 } 4657 4658 4659 void add_exclude(char *target) 4660 { 4661 4662 if(target[0] == '/' || strncmp(target, "./", 2) == 0 || 4663 strncmp(target, "../", 3) == 0) 4664 BAD_ERROR("/, ./ and ../ prefixed excludes not supported with " 4665 "-wildcards or -regex options\n"); 4666 else if(strncmp(target, "... ", 4) == 0) 4667 stickypath = add_path(stickypath, target + 4, target + 4); 4668 else 4669 path = add_path(path, target, target); 4670 } 4671 4672 4673 void display_path(int depth, struct pathname *paths) 4674 { 4675 int i, n; 4676 4677 if(paths == NULL) 4678 return; 4679 4680 for(i = 0; i < paths->names; i++) { 4681 for(n = 0; n < depth; n++) 4682 printf("\t"); 4683 printf("%d: %s\n", depth, paths->name[i].name); 4684 display_path(depth + 1, paths->name[i].paths); 4685 } 4686 } 4687 4688 4689 void display_path2(struct pathname *paths, char *string) 4690 { 4691 int i; 4692 char *path; 4693 4694 if(paths == NULL) { 4695 printf("%s\n", string); 4696 return; 4697 } 4698 4699 for(i = 0; i < paths->names; i++) { 4700 int res = asprintf(&path, "%s/%s", string, paths->name[i].name); 4701 if(res == -1) 4702 BAD_ERROR("asprintf failed in display_path2\n"); 4703 display_path2(paths->name[i].paths, path); 4704 free(path); 4705 } 4706 } 4707 4708 4709 struct pathnames *add_subdir(struct pathnames *paths, struct pathname *path) 4710 { 4711 int count = paths == NULL ? 0 : paths->count; 4712 4713 if(count % PATHS_ALLOC_SIZE == 0) { 4714 paths = realloc(paths, sizeof(struct pathnames) + 4715 (count + PATHS_ALLOC_SIZE) * sizeof(struct pathname *)); 4716 if(paths == NULL) 4717 MEM_ERROR(); 4718 } 4719 4720 paths->path[count] = path; 4721 paths->count = count + 1; 4722 return paths; 4723 } 4724 4725 4726 int excluded_match(char *name, struct pathname *path, struct pathnames **new) 4727 { 4728 int i; 4729 4730 for(i = 0; i < path->names; i++) { 4731 int match = use_regex ? 4732 regexec(path->name[i].preg, name, (size_t) 0, 4733 NULL, 0) == 0 : 4734 fnmatch(path->name[i].name, name, 4735 FNM_PATHNAME|FNM_PERIOD|FNM_EXTMATCH) == 0; 4736 4737 if(match) { 4738 if(path->name[i].paths == NULL || new == NULL) 4739 /* match on a leaf component, any subdirectories 4740 * in the filesystem should be excluded */ 4741 return TRUE; 4742 else 4743 /* match on a non-leaf component, add any 4744 * subdirectories to the new set of 4745 * subdirectories to scan for this name */ 4746 *new = add_subdir(*new, path->name[i].paths); 4747 } 4748 } 4749 4750 return FALSE; 4751 } 4752 4753 4754 int excluded(char *name, struct pathnames *paths, struct pathnames **new) 4755 { 4756 int n; 4757 4758 if(stickypath && excluded_match(name, stickypath, NULL)) 4759 return TRUE; 4760 4761 for(n = 0; paths && n < paths->count; n++) { 4762 int res = excluded_match(name, paths->path[n], new); 4763 if(res) { 4764 free(*new); 4765 *new = NULL; 4766 return TRUE; 4767 } 4768 } 4769 4770 /* 4771 * Either: 4772 * - no matching names found, return empty new search set, or 4773 * - one or more matches with sub-directories found (no leaf matches), 4774 * in which case return new search set. 4775 * 4776 * In either case return FALSE as we don't want to exclude this entry 4777 */ 4778 return FALSE; 4779 } 4780 4781 4782 void process_exclude_file(char *argv) 4783 { 4784 FILE *fd; 4785 char buffer[MAX_LINE + 1]; /* overflow safe */ 4786 char *filename; 4787 4788 fd = fopen(argv, "r"); 4789 if(fd == NULL) 4790 BAD_ERROR("Failed to open exclude file \"%s\" because %s\n", 4791 argv, strerror(errno)); 4792 4793 while(fgets(filename = buffer, MAX_LINE + 1, fd) != NULL) { 4794 int len = strlen(filename); 4795 4796 if(len == MAX_LINE && filename[len - 1] != '\n') 4797 /* line too large */ 4798 BAD_ERROR("Line too long when reading " 4799 "exclude file \"%s\", larger than %d " 4800 "bytes\n", argv, MAX_LINE); 4801 4802 /* 4803 * Remove '\n' terminator if it exists (the last line 4804 * in the file may not be '\n' terminated) 4805 */ 4806 if(len && filename[len - 1] == '\n') 4807 filename[len - 1] = '\0'; 4808 4809 /* Skip any leading whitespace */ 4810 while(isspace(*filename)) 4811 filename ++; 4812 4813 /* if comment line, skip */ 4814 if(*filename == '#') 4815 continue; 4816 4817 /* 4818 * check for initial backslash, to accommodate 4819 * filenames with leading space or leading # character 4820 */ 4821 if(*filename == '\\') 4822 filename ++; 4823 4824 /* if line is now empty after skipping characters, skip it */ 4825 if(*filename == '\0') 4826 continue; 4827 4828 if(old_exclude) 4829 old_add_exclude(filename); 4830 else 4831 add_exclude(filename); 4832 } 4833 4834 if(ferror(fd)) 4835 BAD_ERROR("Reading exclude file \"%s\" failed because %s\n", 4836 argv, strerror(errno)); 4837 4838 fclose(fd); 4839 } 4840 4841 /* ANDROID CHANGES START*/ 4842 #ifdef ANDROID 4843 /* 4844 * Return TRUE (don't compress) if the (regular) file is in the 4845 * whitelist. Else return the Global noD value. 4846 * 4847 * Note : These functions are lifted 100% from the existing exclude 4848 * file code. For maintainability, I've kept this code separate from 4849 * the exclude code instead of having common code for both paths. 4850 */ 4851 static int 4852 whitelisted(struct stat *buf) 4853 { 4854 int i; 4855 4856 /* 4857 * only regular files in the whitelist 4858 */ 4859 if (!S_ISREG(buf->st_mode)) 4860 return noD; 4861 for (i = 0; i < whitelist; i++) { 4862 if ((whitelist_paths[i].st_dev == buf->st_dev) && 4863 (whitelist_paths[i].st_ino == buf->st_ino)) { 4864 /* Don't compress */ 4865 whitelisted_count++; 4866 return TRUE; 4867 } 4868 } 4869 return noD; 4870 } 4871 4872 static void 4873 add_whitelist_entry(char *filename, struct stat *buf) 4874 { 4875 if (!S_ISREG(buf->st_mode)) { 4876 BAD_ERROR("Cannot whitelist %s only regular files can be whitelisted", 4877 filename); 4878 } 4879 if (whitelist % WHITELIST_SIZE == 0) { 4880 whitelist_paths = realloc(whitelist_paths, 4881 (whitelist + WHITELIST_SIZE) 4882 * sizeof(struct whitelist_info)); 4883 if (whitelist_paths == NULL) 4884 MEM_ERROR(); 4885 } 4886 whitelist_paths[whitelist].st_dev = buf->st_dev; 4887 whitelist_paths[whitelist++].st_ino = buf->st_ino; 4888 } 4889 4890 static int 4891 add_whitelist(char *path) 4892 { 4893 int i; 4894 char *filename; 4895 struct stat buf; 4896 4897 /* Absolute of (filesystem) relative path */ 4898 if (path[0] == '/' || strncmp(path, "./", 2) == 0 || 4899 strncmp(path, "../", 3) == 0) { 4900 if(lstat(path, &buf) == -1) { 4901 BAD_ERROR("Cannot stat whitelist dir/file %s because " 4902 "%s", path, strerror(errno)); 4903 } 4904 add_whitelist_entry(path, &buf); 4905 return TRUE; 4906 } 4907 4908 /* pathname relative to mksquashfs source dirs */ 4909 for(i = 0; i < source; i++) { 4910 int res = asprintf(&filename, "%s/%s", source_path[i], path); 4911 if(res == -1) 4912 BAD_ERROR("asprintf failed in add_whitelist\n"); 4913 if(lstat(filename, &buf) == -1) { 4914 if(!(errno == ENOENT || errno == ENOTDIR)) { 4915 BAD_ERROR("Cannot stat whitelist dir/file %s " 4916 "because %s", filename, strerror(errno)); 4917 } 4918 free(filename); 4919 continue; 4920 } 4921 add_whitelist_entry(filename, &buf); 4922 free(filename); 4923 } 4924 return TRUE; 4925 } 4926 4927 static void 4928 process_whitelist_file(char *argv) 4929 { 4930 FILE *fd; 4931 char buffer[MAX_LINE + 1]; /* overflow safe */ 4932 char *filename; 4933 4934 fd = fopen(argv, "r"); 4935 if(fd == NULL) 4936 BAD_ERROR("Failed to open whitelist file \"%s\" because %s\n", 4937 argv, strerror(errno)); 4938 4939 while(fgets(filename = buffer, MAX_LINE + 1, fd) != NULL) { 4940 int len = strlen(filename); 4941 4942 if(len == MAX_LINE && filename[len - 1] != '\n') 4943 /* line too large */ 4944 BAD_ERROR("Line too long when reading " 4945 "whitelist file \"%s\", larger than %d " 4946 "bytes\n", argv, MAX_LINE); 4947 4948 /* 4949 * Remove '\n' terminator if it exists (the last line 4950 * in the file may not be '\n' terminated) 4951 */ 4952 if(len && filename[len - 1] == '\n') 4953 filename[len - 1] = '\0'; 4954 4955 /* Skip any leading whitespace */ 4956 while(isspace(*filename)) 4957 filename ++; 4958 4959 /* if comment line, skip */ 4960 if(*filename == '#') 4961 continue; 4962 4963 /* 4964 * check for initial backslash, to accommodate 4965 * filenames with leading space or leading # character 4966 */ 4967 if(*filename == '\\') 4968 filename ++; 4969 4970 /* if line is now empty after skipping characters, skip it */ 4971 if(*filename == '\0') 4972 continue; 4973 4974 add_whitelist(filename); 4975 } 4976 4977 if(ferror(fd)) 4978 BAD_ERROR("Reading whitelist file \"%s\" failed because %s\n", 4979 argv, strerror(errno)); 4980 4981 fclose(fd); 4982 } 4983 #endif 4984 /* ANDROID CHANGES END */ 4985 4986 #define RECOVER_ID "Squashfs recovery file v1.0\n" 4987 #define RECOVER_ID_SIZE 28 4988 4989 void write_recovery_data(struct squashfs_super_block *sBlk) 4990 { 4991 int res, recoverfd, bytes = sBlk->bytes_used - sBlk->inode_table_start; 4992 pid_t pid = getpid(); 4993 char *metadata; 4994 char header[] = RECOVER_ID; 4995 4996 if(recover == FALSE) { 4997 printf("No recovery data option specified.\n"); 4998 printf("Skipping saving recovery file.\n\n"); 4999 return; 5000 } 5001 5002 metadata = malloc(bytes); 5003 if(metadata == NULL) 5004 MEM_ERROR(); 5005 5006 res = read_fs_bytes(fd, sBlk->inode_table_start, bytes, metadata); 5007 if(res == 0) { 5008 ERROR("Failed to read append filesystem metadata\n"); 5009 BAD_ERROR("Filesystem corrupted?\n"); 5010 } 5011 5012 res = asprintf(&recovery_file, "squashfs_recovery_%s_%d", 5013 getbase(destination_file), pid); 5014 if(res == -1) 5015 MEM_ERROR(); 5016 5017 recoverfd = open(recovery_file, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU); 5018 if(recoverfd == -1) 5019 BAD_ERROR("Failed to create recovery file, because %s. " 5020 "Aborting\n", strerror(errno)); 5021 5022 if(write_bytes(recoverfd, header, RECOVER_ID_SIZE) == -1) 5023 BAD_ERROR("Failed to write recovery file, because %s\n", 5024 strerror(errno)); 5025 5026 if(write_bytes(recoverfd, sBlk, sizeof(struct squashfs_super_block)) == -1) 5027 BAD_ERROR("Failed to write recovery file, because %s\n", 5028 strerror(errno)); 5029 5030 if(write_bytes(recoverfd, metadata, bytes) == -1) 5031 BAD_ERROR("Failed to write recovery file, because %s\n", 5032 strerror(errno)); 5033 5034 close(recoverfd); 5035 free(metadata); 5036 5037 printf("Recovery file \"%s\" written\n", recovery_file); 5038 printf("If Mksquashfs aborts abnormally (i.e. power failure), run\n"); 5039 printf("mksquashfs dummy %s -recover %s\n", destination_file, 5040 recovery_file); 5041 printf("to restore filesystem\n\n"); 5042 } 5043 5044 5045 void read_recovery_data(char *recovery_file, char *destination_file) 5046 { 5047 int fd, recoverfd, bytes; 5048 struct squashfs_super_block orig_sBlk, sBlk; 5049 char *metadata; 5050 int res; 5051 struct stat buf; 5052 char header[] = RECOVER_ID; 5053 char header2[RECOVER_ID_SIZE]; 5054 5055 recoverfd = open(recovery_file, O_RDONLY); 5056 if(recoverfd == -1) 5057 BAD_ERROR("Failed to open recovery file because %s\n", 5058 strerror(errno)); 5059 5060 if(stat(destination_file, &buf) == -1) 5061 BAD_ERROR("Failed to stat destination file, because %s\n", 5062 strerror(errno)); 5063 5064 fd = open(destination_file, O_RDWR); 5065 if(fd == -1) 5066 BAD_ERROR("Failed to open destination file because %s\n", 5067 strerror(errno)); 5068 5069 res = read_bytes(recoverfd, header2, RECOVER_ID_SIZE); 5070 if(res == -1) 5071 BAD_ERROR("Failed to read recovery file, because %s\n", 5072 strerror(errno)); 5073 if(res < RECOVER_ID_SIZE) 5074 BAD_ERROR("Recovery file appears to be truncated\n"); 5075 if(strncmp(header, header2, RECOVER_ID_SIZE) !=0 ) 5076 BAD_ERROR("Not a recovery file\n"); 5077 5078 res = read_bytes(recoverfd, &sBlk, sizeof(struct squashfs_super_block)); 5079 if(res == -1) 5080 BAD_ERROR("Failed to read recovery file, because %s\n", 5081 strerror(errno)); 5082 if(res < sizeof(struct squashfs_super_block)) 5083 BAD_ERROR("Recovery file appears to be truncated\n"); 5084 5085 res = read_fs_bytes(fd, 0, sizeof(struct squashfs_super_block), &orig_sBlk); 5086 if(res == 0) { 5087 ERROR("Failed to read superblock from output filesystem\n"); 5088 BAD_ERROR("Output filesystem is empty!\n"); 5089 } 5090 5091 if(memcmp(((char *) &sBlk) + 4, ((char *) &orig_sBlk) + 4, 5092 sizeof(struct squashfs_super_block) - 4) != 0) 5093 BAD_ERROR("Recovery file and destination file do not seem to " 5094 "match\n"); 5095 5096 bytes = sBlk.bytes_used - sBlk.inode_table_start; 5097 5098 metadata = malloc(bytes); 5099 if(metadata == NULL) 5100 MEM_ERROR(); 5101 5102 res = read_bytes(recoverfd, metadata, bytes); 5103 if(res == -1) 5104 BAD_ERROR("Failed to read recovery file, because %s\n", 5105 strerror(errno)); 5106 if(res < bytes) 5107 BAD_ERROR("Recovery file appears to be truncated\n"); 5108 5109 write_destination(fd, 0, sizeof(struct squashfs_super_block), &sBlk); 5110 5111 write_destination(fd, sBlk.inode_table_start, bytes, metadata); 5112 5113 close(recoverfd); 5114 close(fd); 5115 5116 printf("Successfully wrote recovery file \"%s\". Exiting\n", 5117 recovery_file); 5118 5119 exit(0); 5120 } 5121 5122 5123 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad) 5124 { 5125 int i; 5126 5127 sBlk->fragments = fragments; 5128 sBlk->no_ids = id_count; 5129 sBlk->inode_table_start = write_inodes(); 5130 sBlk->directory_table_start = write_directories(); 5131 sBlk->fragment_table_start = write_fragment_table(); 5132 sBlk->lookup_table_start = exportable ? write_inode_lookup_table() : 5133 SQUASHFS_INVALID_BLK; 5134 sBlk->id_table_start = write_id_table(); 5135 sBlk->xattr_id_table_start = write_xattrs(); 5136 5137 TRACE("sBlk->inode_table_start 0x%llx\n", sBlk->inode_table_start); 5138 TRACE("sBlk->directory_table_start 0x%llx\n", 5139 sBlk->directory_table_start); 5140 TRACE("sBlk->fragment_table_start 0x%llx\n", sBlk->fragment_table_start); 5141 if(exportable) 5142 TRACE("sBlk->lookup_table_start 0x%llx\n", 5143 sBlk->lookup_table_start); 5144 5145 sBlk->bytes_used = bytes; 5146 5147 sBlk->compression = comp->id; 5148 5149 SQUASHFS_INSWAP_SUPER_BLOCK(sBlk); 5150 write_destination(fd, SQUASHFS_START, sizeof(*sBlk), sBlk); 5151 5152 if(!nopad && (i = bytes & (4096 - 1))) { 5153 char temp[4096] = {0}; 5154 write_destination(fd, bytes, 4096 - i, temp); 5155 } 5156 5157 close(fd); 5158 5159 if(recovery_file) 5160 unlink(recovery_file); 5161 5162 total_bytes += total_inode_bytes + total_directory_bytes + 5163 sizeof(struct squashfs_super_block) + total_xattr_bytes; 5164 5165 printf("\n%sSquashfs %d.%d filesystem, %s compressed, data block size" 5166 " %d\n", exportable ? "Exportable " : "", SQUASHFS_MAJOR, 5167 SQUASHFS_MINOR, comp->name, block_size); 5168 printf("\t%s data, %s metadata, %s fragments, %s xattrs\n", 5169 noD ? "uncompressed" : "compressed", noI ? "uncompressed" : 5170 "compressed", no_fragments ? "no" : noF ? "uncompressed" : 5171 "compressed", no_xattrs ? "no" : noX ? "uncompressed" : 5172 "compressed"); 5173 printf("\tduplicates are %sremoved\n", duplicate_checking ? "" : 5174 "not "); 5175 printf("Filesystem size %.2f Kbytes (%.2f Mbytes)\n", bytes / 1024.0, 5176 bytes / (1024.0 * 1024.0)); 5177 printf("\t%.2f%% of uncompressed filesystem size (%.2f Kbytes)\n", 5178 ((float) bytes / total_bytes) * 100.0, total_bytes / 1024.0); 5179 printf("Inode table size %d bytes (%.2f Kbytes)\n", 5180 inode_bytes, inode_bytes / 1024.0); 5181 printf("\t%.2f%% of uncompressed inode table size (%d bytes)\n", 5182 ((float) inode_bytes / total_inode_bytes) * 100.0, 5183 total_inode_bytes); 5184 printf("Directory table size %d bytes (%.2f Kbytes)\n", 5185 directory_bytes, directory_bytes / 1024.0); 5186 printf("\t%.2f%% of uncompressed directory table size (%d bytes)\n", 5187 ((float) directory_bytes / total_directory_bytes) * 100.0, 5188 total_directory_bytes); 5189 if(total_xattr_bytes) { 5190 printf("Xattr table size %d bytes (%.2f Kbytes)\n", 5191 xattr_bytes, xattr_bytes / 1024.0); 5192 printf("\t%.2f%% of uncompressed xattr table size (%d bytes)\n", 5193 ((float) xattr_bytes / total_xattr_bytes) * 100.0, 5194 total_xattr_bytes); 5195 } 5196 if(duplicate_checking) 5197 printf("Number of duplicate files found %d\n", file_count - 5198 dup_files); 5199 else 5200 printf("No duplicate files removed\n"); 5201 printf("Number of inodes %d\n", inode_count); 5202 printf("Number of files %d\n", file_count); 5203 if(!no_fragments) 5204 printf("Number of fragments %d\n", fragments); 5205 printf("Number of symbolic links %d\n", sym_count); 5206 printf("Number of device nodes %d\n", dev_count); 5207 printf("Number of fifo nodes %d\n", fifo_count); 5208 printf("Number of socket nodes %d\n", sock_count); 5209 printf("Number of directories %d\n", dir_count); 5210 printf("Number of ids (unique uids + gids) %d\n", id_count); 5211 printf("Number of uids %d\n", uid_count); 5212 5213 for(i = 0; i < id_count; i++) { 5214 if(id_table[i]->flags & ISA_UID) { 5215 struct passwd *user = getpwuid(id_table[i]->id); 5216 printf("\t%s (%d)\n", user == NULL ? "unknown" : 5217 user->pw_name, id_table[i]->id); 5218 } 5219 } 5220 5221 printf("Number of gids %d\n", guid_count); 5222 5223 for(i = 0; i < id_count; i++) { 5224 if(id_table[i]->flags & ISA_GID) { 5225 struct group *group = getgrgid(id_table[i]->id); 5226 printf("\t%s (%d)\n", group == NULL ? "unknown" : 5227 group->gr_name, id_table[i]->id); 5228 } 5229 } 5230 5231 printf("Number of whitelisted (uncompressed) files %d\n", 5232 whitelisted_count); 5233 } 5234 5235 5236 int parse_numberll(char *start, long long *res, int size) 5237 { 5238 char *end; 5239 long long number; 5240 5241 errno = 0; /* To distinguish success/failure after call */ 5242 5243 number = strtoll(start, &end, 10); 5244 5245 /* 5246 * check for strtoll underflow or overflow in conversion, and other 5247 * errors. 5248 */ 5249 if((errno == ERANGE && (number == LLONG_MIN || number == LLONG_MAX)) || 5250 (errno != 0 && number == 0)) 5251 return 0; 5252 5253 /* reject negative numbers as invalid */ 5254 if(number < 0) 5255 return 0; 5256 5257 if(size) { 5258 /* 5259 * Check for multiplier and trailing junk. 5260 * But first check that a number exists before the 5261 * multiplier 5262 */ 5263 if(end == start) 5264 return 0; 5265 5266 switch(end[0]) { 5267 case 'g': 5268 case 'G': 5269 if(multiply_overflowll(number, 1073741824)) 5270 return 0; 5271 number *= 1073741824; 5272 5273 if(end[1] != '\0') 5274 /* trailing junk after multiplier, but 5275 * allow it to be "bytes" */ 5276 if(strcmp(end + 1, "bytes")) 5277 return 0; 5278 5279 break; 5280 case 'm': 5281 case 'M': 5282 if(multiply_overflowll(number, 1048576)) 5283 return 0; 5284 number *= 1048576; 5285 5286 if(end[1] != '\0') 5287 /* trailing junk after multiplier, but 5288 * allow it to be "bytes" */ 5289 if(strcmp(end + 1, "bytes")) 5290 return 0; 5291 5292 break; 5293 case 'k': 5294 case 'K': 5295 if(multiply_overflowll(number, 1024)) 5296 return 0; 5297 number *= 1024; 5298 5299 if(end[1] != '\0') 5300 /* trailing junk after multiplier, but 5301 * allow it to be "bytes" */ 5302 if(strcmp(end + 1, "bytes")) 5303 return 0; 5304 5305 break; 5306 case '\0': 5307 break; 5308 default: 5309 /* trailing junk after number */ 5310 return 0; 5311 } 5312 } else if(end[0] != '\0') 5313 /* trailing junk after number */ 5314 return 0; 5315 5316 *res = number; 5317 return 1; 5318 } 5319 5320 5321 int parse_number(char *start, int *res, int size) 5322 { 5323 long long number; 5324 5325 if(!parse_numberll(start, &number, size)) 5326 return 0; 5327 5328 /* check if long result will overflow signed int */ 5329 if(number > INT_MAX) 5330 return 0; 5331 5332 *res = (int) number; 5333 return 1; 5334 } 5335 5336 5337 int parse_num(char *arg, int *res) 5338 { 5339 return parse_number(arg, res, 0); 5340 } 5341 5342 5343 int parse_ugid_map(char *map_str, 5344 struct ugid_map_entry ugid_mapping[UGID_ENTRIES], 5345 unsigned int *ugid_map_count) 5346 { 5347 char *line_state, *token_state; 5348 char *line, *line_str, *token, *token_str; 5349 long long numbers[3]; 5350 int i; 5351 5352 for (*ugid_map_count = 0, line_str = map_str;; 5353 ++*ugid_map_count, line_str = NULL) { 5354 line = strtok_r(line_str, "\n", &line_state); 5355 if (line == NULL) 5356 break; 5357 ERROR("line: %s\n", line); 5358 if (*ugid_map_count >= UGID_ENTRIES) { 5359 ERROR("Too many entries for u/gid mapping\n"); 5360 return -1; 5361 } 5362 5363 for (i = 0, token_str = line; i < 3; i++, token_str = NULL) { 5364 token = strtok_r(token_str, " ", &token_state); 5365 ERROR("token: %d, %s\n", i, token); 5366 if (token == NULL || 5367 !parse_numberll(token, &numbers[i], 0) || 5368 numbers[i] < 0 || numbers[i] > ULONG_MAX) { 5369 ERROR("Malformed u/gid mapping line1\n"); 5370 return -1; 5371 } 5372 } 5373 5374 if (numbers[0] + numbers[2] > ULONG_MAX) { 5375 ERROR("u/gid mapping overflow\n"); 5376 return -1; 5377 } 5378 5379 if (numbers[1] + numbers[2] > ULONG_MAX) { 5380 ERROR("u/gid mapping overflow\n"); 5381 return -1; 5382 } 5383 5384 if (strtok_r(NULL, " ", &token_state) != NULL) { 5385 ERROR("Malformed u/gid mapping line2\n"); 5386 return -1; 5387 } 5388 5389 ugid_mapping[*ugid_map_count].child_id = 5390 (unsigned int)numbers[0]; 5391 ugid_mapping[*ugid_map_count].parent_id = 5392 (unsigned int)numbers[1]; 5393 ugid_mapping[*ugid_map_count].length = (unsigned int)numbers[2]; 5394 } 5395 5396 return 0; 5397 } 5398 5399 5400 int get_physical_memory() 5401 { 5402 int phys_mem; 5403 #ifndef linux 5404 #ifdef HW_MEMSIZE 5405 #define SYSCTL_PHYSMEM HW_MEMSIZE 5406 #elif defined(HW_PHYSMEM64) 5407 #define SYSCTL_PHYSMEM HW_PHYSMEM64 5408 #else 5409 #define SYSCTL_PHYSMEM HW_PHYSMEM 5410 #endif 5411 5412 int mib[2]; 5413 uint64_t sysctl_physmem = 0; 5414 size_t sysctl_len = sizeof(sysctl_physmem); 5415 5416 mib[0] = CTL_HW; 5417 mib[1] = SYSCTL_PHYSMEM; 5418 5419 if(sysctl(mib, 2, &sysctl_physmem, &sysctl_len, NULL, 0) == 0) { 5420 /* some systems use 32-bit values, work with what we're given */ 5421 if (sysctl_len == 4) 5422 sysctl_physmem = *(uint32_t*)&sysctl_physmem; 5423 phys_mem = sysctl_physmem >> 20; 5424 } else { 5425 ERROR_START("Failed to get amount of available " 5426 "memory."); 5427 ERROR_EXIT(" Defaulting to least viable amount\n"); 5428 phys_mem = SQUASHFS_LOWMEM; 5429 } 5430 #undef SYSCTL_PHYSMEM 5431 #else 5432 /* Long longs are used here because with PAE, a 32-bit 5433 machine can have more than 4GB of physical memory */ 5434 5435 long long num_pages = sysconf(_SC_PHYS_PAGES); 5436 long long page_size = sysconf(_SC_PAGESIZE); 5437 phys_mem = num_pages * page_size >> 20; 5438 if(num_pages == -1 || page_size == -1) 5439 return 0; 5440 5441 #endif 5442 5443 if(phys_mem < SQUASHFS_LOWMEM) 5444 BAD_ERROR("Mksquashfs requires more physical memory than is " 5445 "available!\n"); 5446 5447 return phys_mem; 5448 } 5449 5450 5451 void check_usable_phys_mem(int total_mem) 5452 { 5453 /* 5454 * We want to allow users to use as much of their physical 5455 * memory as they wish. However, for practical reasons there are 5456 * limits which need to be imposed, to protect users from themselves 5457 * and to prevent people from using Mksquashfs as a DOS attack by using 5458 * all physical memory. Mksquashfs uses memory to cache data from disk 5459 * to optimise performance. It is pointless to ask it to use more 5460 * than 75% of physical memory, as this causes thrashing and it is thus 5461 * self-defeating. 5462 */ 5463 int mem = get_physical_memory(); 5464 5465 mem = (mem >> 1) + (mem >> 2); /* 75% */ 5466 5467 if(total_mem > mem && mem) { 5468 ERROR("Total memory requested is more than 75%% of physical " 5469 "memory.\n"); 5470 ERROR("Mksquashfs uses memory to cache data from disk to " 5471 "optimise performance.\n"); 5472 ERROR("It is pointless to ask it to use more than this amount " 5473 "of memory, as this\n"); 5474 ERROR("causes thrashing and it is thus self-defeating.\n"); 5475 BAD_ERROR("Requested memory size too large\n"); 5476 } 5477 5478 if(sizeof(void *) == 4 && total_mem > 2048) { 5479 /* 5480 * If we're running on a kernel with PAE or on a 64-bit kernel, 5481 * then the 75% physical memory limit can still easily exceed 5482 * the addressable memory by this process. 5483 * 5484 * Due to the typical kernel/user-space split (1GB/3GB, or 5485 * 2GB/2GB), we have to conservatively assume the 32-bit 5486 * processes can only address 2-3GB. So refuse if the user 5487 * tries to allocate more than 2GB. 5488 */ 5489 ERROR("Total memory requested may exceed maximum " 5490 "addressable memory by this process\n"); 5491 BAD_ERROR("Requested memory size too large\n"); 5492 } 5493 } 5494 5495 5496 int get_default_phys_mem() 5497 { 5498 /* 5499 * get_physical_memory() relies on /proc being mounted. 5500 * If it fails, issue a warning, and use 5501 * SQUASHFS_LOWMEM / SQUASHFS_TAKE as default, 5502 * and allow a larger value to be set with -mem. 5503 */ 5504 int mem = get_physical_memory(); 5505 5506 if(mem == 0) { 5507 mem = SQUASHFS_LOWMEM / SQUASHFS_TAKE; 5508 5509 ERROR("Warning: Cannot get size of physical memory, probably " 5510 "because /proc is missing.\n"); 5511 ERROR("Warning: Defaulting to minimal use of %d Mbytes, use " 5512 "-mem to set a better value,\n", mem); 5513 ERROR("Warning: or fix /proc.\n"); 5514 } else 5515 mem /= SQUASHFS_TAKE; 5516 5517 if(sizeof(void *) == 4 && mem > 640) { 5518 /* 5519 * If we're running on a kernel with PAE or on a 64-bit kernel, 5520 * the default memory usage can exceed the addressable 5521 * memory by this process. 5522 * Due to the typical kernel/user-space split (1GB/3GB, or 5523 * 2GB/2GB), we have to conservatively assume the 32-bit 5524 * processes can only address 2-3GB. So limit the default 5525 * usage to 640M, which gives room for other data. 5526 */ 5527 mem = 640; 5528 } 5529 5530 return mem; 5531 } 5532 5533 5534 void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq, 5535 int *fwriteq) 5536 { 5537 *readq = mem / SQUASHFS_READQ_MEM; 5538 *bwriteq = mem / SQUASHFS_BWRITEQ_MEM; 5539 *fwriteq = mem / SQUASHFS_FWRITEQ_MEM; 5540 *fragq = mem - *readq - *bwriteq - *fwriteq; 5541 } 5542 5543 5544 #define VERSION() \ 5545 printf("mksquashfs version 4.3-git (2014/09/12)\n");\ 5546 printf("copyright (C) 2014 Phillip Lougher "\ 5547 "<phillip (at) squashfs.org.uk>\n\n"); \ 5548 printf("This program is free software; you can redistribute it and/or"\ 5549 "\n");\ 5550 printf("modify it under the terms of the GNU General Public License"\ 5551 "\n");\ 5552 printf("as published by the Free Software Foundation; either version "\ 5553 "2,\n");\ 5554 printf("or (at your option) any later version.\n\n");\ 5555 printf("This program is distributed in the hope that it will be "\ 5556 "useful,\n");\ 5557 printf("but WITHOUT ANY WARRANTY; without even the implied warranty "\ 5558 "of\n");\ 5559 printf("MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the"\ 5560 "\n");\ 5561 printf("GNU General Public License for more details.\n"); 5562 int main(int argc, char *argv[]) 5563 { 5564 struct stat buf, source_buf; 5565 int res, i; 5566 char *b, *root_name = NULL; 5567 int keep_as_directory = FALSE; 5568 squashfs_inode inode; 5569 int readq; 5570 int fragq; 5571 int bwriteq; 5572 int fwriteq; 5573 int total_mem = get_default_phys_mem(); 5574 int progress = TRUE; 5575 int force_progress = FALSE; 5576 struct file_buffer **fragment = NULL; 5577 /* ANDROID CHANGES START*/ 5578 #ifdef ANDROID 5579 const char *fs_config_file = NULL; 5580 #endif 5581 /* ANDROID CHANGES END */ 5582 5583 if(argc > 1 && strcmp(argv[1], "-version") == 0) { 5584 VERSION(); 5585 exit(0); 5586 } 5587 5588 block_log = slog(block_size); 5589 calculate_queue_sizes(total_mem, &readq, &fragq, &bwriteq, &fwriteq); 5590 5591 for(i = 1; i < argc && argv[i][0] != '-'; i++); 5592 if(i < 3) 5593 goto printOptions; 5594 source_path = argv + 1; 5595 source = i - 2; 5596 5597 /* 5598 * Scan the command line for -comp xxx option, this is to ensure 5599 * any -X compressor specific options are passed to the 5600 * correct compressor 5601 */ 5602 for(; i < argc; i++) { 5603 struct compressor *prev_comp = comp; 5604 5605 if(strcmp(argv[i], "-comp") == 0) { 5606 if(++i == argc) { 5607 ERROR("%s: -comp missing compression type\n", 5608 argv[0]); 5609 exit(1); 5610 } 5611 comp = lookup_compressor(argv[i]); 5612 if(!comp->supported) { 5613 ERROR("%s: Compressor \"%s\" is not supported!" 5614 "\n", argv[0], argv[i]); 5615 ERROR("%s: Compressors available:\n", argv[0]); 5616 display_compressors("", COMP_DEFAULT); 5617 exit(1); 5618 } 5619 if(prev_comp != NULL && prev_comp != comp) { 5620 ERROR("%s: -comp multiple conflicting -comp" 5621 " options specified on command line" 5622 ", previously %s, now %s\n", argv[0], 5623 prev_comp->name, comp->name); 5624 exit(1); 5625 } 5626 compressor_opt_parsed = 1; 5627 5628 } else if(strcmp(argv[i], "-e") == 0) 5629 break; 5630 else if(strcmp(argv[i], "-root-becomes") == 0 || 5631 strcmp(argv[i], "-ef") == 0 || 5632 strcmp(argv[i], "-pf") == 0 || 5633 strcmp(argv[i], "-vaf") == 0 || 5634 strcmp(argv[i], "-comp") == 0) 5635 i++; 5636 } 5637 5638 /* 5639 * if no -comp option specified lookup default compressor. Note the 5640 * Makefile ensures the default compressor has been built, and so we 5641 * don't need to to check for failure here 5642 */ 5643 if(comp == NULL) 5644 comp = lookup_compressor(COMP_DEFAULT); 5645 5646 for(i = source + 2; i < argc; i++) { 5647 if(strcmp(argv[i], "-action") == 0 || 5648 strcmp(argv[i], "-a") ==0) { 5649 if(++i == argc) { 5650 ERROR("%s: %s missing action\n", 5651 argv[0], argv[i - 1]); 5652 exit(1); 5653 } 5654 res = parse_action(argv[i], ACTION_LOG_NONE); 5655 if(res == 0) 5656 exit(1); 5657 5658 } else if(strcmp(argv[i], "-verbose-action") == 0 || 5659 strcmp(argv[i], "-va") ==0) { 5660 if(++i == argc) { 5661 ERROR("%s: %s missing action\n", 5662 argv[0], argv[i - 1]); 5663 exit(1); 5664 } 5665 res = parse_action(argv[i], ACTION_LOG_VERBOSE); 5666 if(res == 0) 5667 exit(1); 5668 5669 } else if(strcmp(argv[i], "-true-action") == 0 || 5670 strcmp(argv[i], "-ta") ==0) { 5671 if(++i == argc) { 5672 ERROR("%s: %s missing action\n", 5673 argv[0], argv[i - 1]); 5674 exit(1); 5675 } 5676 res = parse_action(argv[i], ACTION_LOG_TRUE); 5677 if(res == 0) 5678 exit(1); 5679 5680 } else if(strcmp(argv[i], "-false-action") == 0 || 5681 strcmp(argv[i], "-fa") ==0) { 5682 if(++i == argc) { 5683 ERROR("%s: %s missing action\n", 5684 argv[0], argv[i - 1]); 5685 exit(1); 5686 } 5687 res = parse_action(argv[i], ACTION_LOG_FALSE); 5688 if(res == 0) 5689 exit(1); 5690 5691 } else if(strcmp(argv[i], "-action-file") == 0 || 5692 strcmp(argv[i], "-af") ==0) { 5693 if(++i == argc) { 5694 ERROR("%s: %s missing filename\n", argv[0], 5695 argv[i - 1]); 5696 exit(1); 5697 } 5698 if(read_action_file(argv[i], ACTION_LOG_NONE) == FALSE) 5699 exit(1); 5700 5701 } else if(strcmp(argv[i], "-verbose-action-file") == 0 || 5702 strcmp(argv[i], "-vaf") ==0) { 5703 if(++i == argc) { 5704 ERROR("%s: %s missing filename\n", argv[0], 5705 argv[i - 1]); 5706 exit(1); 5707 } 5708 if(read_action_file(argv[i], ACTION_LOG_VERBOSE) == FALSE) 5709 exit(1); 5710 5711 } else if(strcmp(argv[i], "-true-action-file") == 0 || 5712 strcmp(argv[i], "-taf") ==0) { 5713 if(++i == argc) { 5714 ERROR("%s: %s missing filename\n", argv[0], 5715 argv[i - 1]); 5716 exit(1); 5717 } 5718 if(read_action_file(argv[i], ACTION_LOG_TRUE) == FALSE) 5719 exit(1); 5720 5721 } else if(strcmp(argv[i], "-false-action-file") == 0 || 5722 strcmp(argv[i], "-faf") ==0) { 5723 if(++i == argc) { 5724 ERROR("%s: %s missing filename\n", argv[0], 5725 argv[i - 1]); 5726 exit(1); 5727 } 5728 if(read_action_file(argv[i], ACTION_LOG_FALSE) == FALSE) 5729 exit(1); 5730 5731 } else if(strcmp(argv[i], "-comp") == 0) 5732 /* parsed previously */ 5733 i++; 5734 5735 else if(strncmp(argv[i], "-X", 2) == 0) { 5736 int args; 5737 5738 if(strcmp(argv[i] + 2, "help") == 0) 5739 goto print_compressor_options; 5740 5741 args = compressor_options(comp, argv + i, argc - i); 5742 if(args < 0) { 5743 if(args == -1) { 5744 ERROR("%s: Unrecognised compressor" 5745 " option %s\n", argv[0], 5746 argv[i]); 5747 if(!compressor_opt_parsed) 5748 ERROR("%s: Did you forget to" 5749 " specify -comp?\n", 5750 argv[0]); 5751 print_compressor_options: 5752 ERROR("%s: selected compressor \"%s\"" 5753 ". Options supported: %s\n", 5754 argv[0], comp->name, 5755 comp->usage ? "" : "none"); 5756 if(comp->usage) 5757 comp->usage(); 5758 } 5759 exit(1); 5760 } 5761 i += args; 5762 5763 } else if(strcmp(argv[i], "-pf") == 0) { 5764 if(++i == argc) { 5765 ERROR("%s: -pf missing filename\n", argv[0]); 5766 exit(1); 5767 } 5768 if(read_pseudo_file(argv[i]) == FALSE) 5769 exit(1); 5770 } else if(strcmp(argv[i], "-p") == 0) { 5771 if(++i == argc) { 5772 ERROR("%s: -p missing pseudo file definition\n", 5773 argv[0]); 5774 exit(1); 5775 } 5776 if(read_pseudo_def(argv[i]) == FALSE) 5777 exit(1); 5778 } else if(strcmp(argv[i], "-recover") == 0) { 5779 if(++i == argc) { 5780 ERROR("%s: -recover missing recovery file\n", 5781 argv[0]); 5782 exit(1); 5783 } 5784 read_recovery_data(argv[i], argv[source + 1]); 5785 } else if(strcmp(argv[i], "-no-recovery") == 0) 5786 recover = FALSE; 5787 else if(strcmp(argv[i], "-wildcards") == 0) { 5788 old_exclude = FALSE; 5789 use_regex = FALSE; 5790 } else if(strcmp(argv[i], "-regex") == 0) { 5791 old_exclude = FALSE; 5792 use_regex = TRUE; 5793 } else if(strcmp(argv[i], "-no-sparse") == 0) 5794 sparse_files = FALSE; 5795 else if(strcmp(argv[i], "-no-progress") == 0) 5796 progress = FALSE; 5797 else if(strcmp(argv[i], "-progress") == 0) 5798 force_progress = TRUE; 5799 else if(strcmp(argv[i], "-no-exports") == 0) 5800 exportable = FALSE; 5801 else if(strcmp(argv[i], "-processors") == 0) { 5802 if((++i == argc) || !parse_num(argv[i], &processors)) { 5803 ERROR("%s: -processors missing or invalid " 5804 "processor number\n", argv[0]); 5805 exit(1); 5806 } 5807 if(processors < 1) { 5808 ERROR("%s: -processors should be 1 or larger\n", 5809 argv[0]); 5810 exit(1); 5811 } 5812 } else if(strcmp(argv[i], "-read-queue") == 0) { 5813 if((++i == argc) || !parse_num(argv[i], &readq)) { 5814 ERROR("%s: -read-queue missing or invalid " 5815 "queue size\n", argv[0]); 5816 exit(1); 5817 } 5818 if(readq < 1) { 5819 ERROR("%s: -read-queue should be 1 megabyte or " 5820 "larger\n", argv[0]); 5821 exit(1); 5822 } 5823 } else if(strcmp(argv[i], "-write-queue") == 0) { 5824 if((++i == argc) || !parse_num(argv[i], &bwriteq)) { 5825 ERROR("%s: -write-queue missing or invalid " 5826 "queue size\n", argv[0]); 5827 exit(1); 5828 } 5829 if(bwriteq < 2) { 5830 ERROR("%s: -write-queue should be 2 megabytes " 5831 "or larger\n", argv[0]); 5832 exit(1); 5833 } 5834 fwriteq = bwriteq >> 1; 5835 bwriteq -= fwriteq; 5836 } else if(strcmp(argv[i], "-fragment-queue") == 0) { 5837 if((++i == argc) || !parse_num(argv[i], &fragq)) { 5838 ERROR("%s: -fragment-queue missing or invalid " 5839 "queue size\n", argv[0]); 5840 exit(1); 5841 } 5842 if(fragq < 1) { 5843 ERROR("%s: -fragment-queue should be 1 " 5844 "megabyte or larger\n", argv[0]); 5845 exit(1); 5846 } 5847 } else if(strcmp(argv[i], "-mem") == 0) { 5848 long long number; 5849 5850 if((++i == argc) || 5851 !parse_numberll(argv[i], &number, 1)) { 5852 ERROR("%s: -mem missing or invalid mem size\n", 5853 argv[0]); 5854 exit(1); 5855 } 5856 5857 /* 5858 * convert from bytes to Mbytes, ensuring the value 5859 * does not overflow a signed int 5860 */ 5861 if(number >= (1LL << 51)) { 5862 ERROR("%s: -mem invalid mem size\n", argv[0]); 5863 exit(1); 5864 } 5865 5866 total_mem = number / 1048576; 5867 if(total_mem < (SQUASHFS_LOWMEM / SQUASHFS_TAKE)) { 5868 ERROR("%s: -mem should be %d Mbytes or " 5869 "larger\n", argv[0], 5870 SQUASHFS_LOWMEM / SQUASHFS_TAKE); 5871 exit(1); 5872 } 5873 calculate_queue_sizes(total_mem, &readq, &fragq, 5874 &bwriteq, &fwriteq); 5875 } else if(strcmp(argv[i], "-b") == 0) { 5876 if(++i == argc) { 5877 ERROR("%s: -b missing block size\n", argv[0]); 5878 exit(1); 5879 } 5880 if(!parse_number(argv[i], &block_size, 1)) { 5881 ERROR("%s: -b invalid block size\n", argv[0]); 5882 exit(1); 5883 } 5884 if((block_log = slog(block_size)) == 0) { 5885 ERROR("%s: -b block size not power of two or " 5886 "not between 4096 and 1Mbyte\n", 5887 argv[0]); 5888 exit(1); 5889 } 5890 } else if(strcmp(argv[i], "-ef") == 0) { 5891 if(++i == argc) { 5892 ERROR("%s: -ef missing filename\n", argv[0]); 5893 exit(1); 5894 } 5895 } else if(strcmp(argv[i], "-no-duplicates") == 0) 5896 duplicate_checking = FALSE; 5897 5898 else if(strcmp(argv[i], "-no-fragments") == 0) 5899 no_fragments = TRUE; 5900 5901 else if(strcmp(argv[i], "-always-use-fragments") == 0) 5902 always_use_fragments = TRUE; 5903 5904 else if(strcmp(argv[i], "-sort") == 0) { 5905 if(++i == argc) { 5906 ERROR("%s: -sort missing filename\n", argv[0]); 5907 exit(1); 5908 } 5909 } else if(strcmp(argv[i], "-all-root") == 0 || 5910 strcmp(argv[i], "-root-owned") == 0) 5911 global_uid = global_gid = 0; 5912 5913 else if(strcmp(argv[i], "-force-uid") == 0) { 5914 if(++i == argc) { 5915 ERROR("%s: -force-uid missing uid or user\n", 5916 argv[0]); 5917 exit(1); 5918 } 5919 if((global_uid = strtoll(argv[i], &b, 10)), *b =='\0') { 5920 if(global_uid < 0 || global_uid > 5921 (((long long) 1 << 32) - 1)) { 5922 ERROR("%s: -force-uid uid out of range" 5923 "\n", argv[0]); 5924 exit(1); 5925 } 5926 } else { 5927 struct passwd *uid = getpwnam(argv[i]); 5928 if(uid) 5929 global_uid = uid->pw_uid; 5930 else { 5931 ERROR("%s: -force-uid invalid uid or " 5932 "unknown user\n", argv[0]); 5933 exit(1); 5934 } 5935 } 5936 } else if(strcmp(argv[i], "-force-gid") == 0) { 5937 if(++i == argc) { 5938 ERROR("%s: -force-gid missing gid or group\n", 5939 argv[0]); 5940 exit(1); 5941 } 5942 if((global_gid = strtoll(argv[i], &b, 10)), *b =='\0') { 5943 if(global_gid < 0 || global_gid > 5944 (((long long) 1 << 32) - 1)) { 5945 ERROR("%s: -force-gid gid out of range" 5946 "\n", argv[0]); 5947 exit(1); 5948 } 5949 } else { 5950 struct group *gid = getgrnam(argv[i]); 5951 if(gid) 5952 global_gid = gid->gr_gid; 5953 else { 5954 ERROR("%s: -force-gid invalid gid or " 5955 "unknown group\n", argv[0]); 5956 exit(1); 5957 } 5958 } 5959 } else if(strcmp(argv[i], "-noI") == 0 || 5960 strcmp(argv[i], "-noInodeCompression") == 0) 5961 noI = TRUE; 5962 5963 else if(strcmp(argv[i], "-noD") == 0 || 5964 strcmp(argv[i], "-noDataCompression") == 0) 5965 noD = TRUE; 5966 5967 else if(strcmp(argv[i], "-noF") == 0 || 5968 strcmp(argv[i], "-noFragmentCompression") == 0) 5969 noF = TRUE; 5970 5971 else if(strcmp(argv[i], "-noX") == 0 || 5972 strcmp(argv[i], "-noXattrCompression") == 0) 5973 noX = TRUE; 5974 5975 else if(strcmp(argv[i], "-no-xattrs") == 0) 5976 no_xattrs = TRUE; 5977 5978 else if(strcmp(argv[i], "-xattrs") == 0) 5979 no_xattrs = FALSE; 5980 5981 /* ANDROID CHANGES START*/ 5982 #ifdef ANDROID 5983 else if(strcmp(argv[i], "-context-file") == 0) { 5984 if(++i == argc) { 5985 ERROR("%s: -context-file: missing file name\n", 5986 argv[0]); 5987 exit(1); 5988 } 5989 context_file = argv[i]; 5990 } 5991 else if(strcmp(argv[i], "-fs-config-file") == 0) { 5992 if(++i == argc) { 5993 ERROR("%s: -fs-config-file: missing file name\n", 5994 argv[0]); 5995 exit(1); 5996 } 5997 fs_config_file = argv[i]; 5998 } else if(strcmp(argv[i], "-whitelist") == 0) { 5999 if(++i == argc) { 6000 ERROR("%s: -whitelist missing filename\n", argv[0]); 6001 exit(1); 6002 } 6003 whitelist_filename = argv[i]; 6004 } 6005 else if(strcmp(argv[i], "-t") == 0) { 6006 if(++i == argc) { 6007 ERROR("%s: -t missing compression threshold percentage\n", argv[0]); 6008 exit(1); 6009 } 6010 if(!parse_number(argv[i], &compress_thresh_per, 1)) { 6011 ERROR("%s: -t invalid compression threshold percentage\n", argv[0]); 6012 exit(1); 6013 } 6014 if(compress_thresh_per > 100 || compress_thresh_per < 0) { 6015 ERROR("%s: -t compression threshold percentage not between 0 and 100\n", 6016 argv[0]); 6017 exit(1); 6018 } 6019 } 6020 #endif 6021 /* ANDROID CHANGES END */ 6022 else if(strcmp(argv[i], "-nopad") == 0) 6023 nopad = TRUE; 6024 6025 else if(strcmp(argv[i], "-info") == 0) 6026 silent = FALSE; 6027 6028 else if(strcmp(argv[i], "-e") == 0) 6029 break; 6030 6031 else if(strcmp(argv[i], "-noappend") == 0) 6032 delete = TRUE; 6033 6034 else if(strcmp(argv[i], "-keep-as-directory") == 0) 6035 keep_as_directory = TRUE; 6036 /* ANDROID CHANGES START*/ 6037 #ifdef ANDROID 6038 else if(strcmp(argv[i], "-android-fs-config") == 0) 6039 android_config = TRUE; 6040 else if(strcmp(argv[i], "-mount-point") == 0) { 6041 if(++i == argc) { 6042 ERROR("%s: -mount-point: missing mount point name\n", 6043 argv[0]); 6044 exit(1); 6045 } 6046 mount_point = argv[i]; 6047 } 6048 else if(strcmp(argv[i], "-product-out") == 0) { 6049 if(++i == argc) { 6050 ERROR("%s: -product-out: missing path name\n", 6051 argv[0]); 6052 exit(1); 6053 } 6054 target_out_path = argv[i]; 6055 } 6056 else if(strcmp(argv[i], "-disable-4k-align") == 0) 6057 align_4k_blocks = FALSE; 6058 else if(strcmp(argv[i], "-block-map") == 0) { 6059 if(++i == argc) { 6060 ERROR("%s: -block-map: missing path name\n", 6061 argv[0]); 6062 exit(1); 6063 } 6064 block_map_file = fopen(argv[i], "w"); 6065 if (block_map_file == NULL) { 6066 ERROR("%s: -block-map: failed to open %s\n", 6067 argv[0], argv[i]); 6068 exit(1); 6069 } 6070 if (!align_4k_blocks) { 6071 ERROR("WARNING: Using block maps with unaligned 4k blocks " 6072 "is not ideal as block map offsets are multiples of 4k, " 6073 "consider not passing -disable-4k-align\n"); 6074 } 6075 } 6076 #endif 6077 /* ANDROID CHANGES END */ 6078 6079 else if(strcmp(argv[i], "-exit-on-error") == 0) 6080 exit_on_error = TRUE; 6081 6082 else if(strcmp(argv[i], "-root-becomes") == 0) { 6083 if(++i == argc) { 6084 ERROR("%s: -root-becomes: missing name\n", 6085 argv[0]); 6086 exit(1); 6087 } 6088 root_name = argv[i]; 6089 } else if (strcmp(argv[i], "-uid-map") == 0) { 6090 if (++i == argc) { 6091 ERROR("%s: -uid-map: missing mapping\n", 6092 argv[0]); 6093 exit(1); 6094 } 6095 if (parse_ugid_map(argv[i], uid_mapping, 6096 &uid_map_count) != 0) { 6097 ERROR("%s: -uid-map: invalid mapping\n", 6098 argv[0]); 6099 exit(1); 6100 } 6101 } else if (strcmp(argv[i], "-gid-map") == 0) { 6102 if (++i == argc) { 6103 ERROR("%s: -gid-map: missing mapping\n", 6104 argv[0]); 6105 exit(1); 6106 } 6107 if (parse_ugid_map(argv[i], gid_mapping, 6108 &gid_map_count) != 0) { 6109 ERROR("%s: -gid-map: invalid mapping\n", 6110 argv[0]); 6111 exit(1); 6112 } 6113 } else if(strcmp(argv[i], "-version") == 0) { 6114 VERSION(); 6115 } else { 6116 ERROR("%s: invalid option\n\n", argv[0]); 6117 printOptions: 6118 ERROR("SYNTAX:%s source1 source2 ... dest [options] " 6119 "[-e list of exclude\ndirs/files]\n", argv[0]); 6120 ERROR("\nFilesystem build options:\n"); 6121 ERROR("-comp <comp>\t\tselect <comp> compression\n"); 6122 ERROR("\t\t\tCompressors available:\n"); 6123 display_compressors("\t\t\t", COMP_DEFAULT); 6124 ERROR("-b <block_size>\t\tset data block to " 6125 "<block_size>. Default 128 Kbytes\n"); 6126 ERROR("\t\t\tOptionally a suffix of K or M can be" 6127 " given to specify\n\t\t\tKbytes or Mbytes" 6128 " respectively\n"); 6129 ERROR("-no-exports\t\tdon't make the filesystem " 6130 "exportable via NFS\n"); 6131 ERROR("-no-sparse\t\tdon't detect sparse files\n"); 6132 ERROR("-no-xattrs\t\tdon't store extended attributes" 6133 NOXOPT_STR "\n"); 6134 ERROR("-xattrs\t\t\tstore extended attributes" XOPT_STR 6135 "\n"); 6136 /* ANDROID CHANGES START*/ 6137 #ifdef ANDROID 6138 ERROR("-context-file <file>\tApply selinux security " 6139 "xattrs from context-file instead\n\t\t\t" 6140 "of reading xattrs from file system\n"); 6141 ERROR("-fs-config-file <file>\tAndroid specific " 6142 "filesystem config file\n"); 6143 ERROR("-t <compress_thresh>\tset minimum " 6144 "acceptable compression ratio of a block to\n\t\t\t" 6145 "<compress_thresh_per> otherwise don't compress. " 6146 "Default 0%\n"); 6147 ERROR("-whitelist <file>\tAndroid specific whitelist " 6148 "one entry per line (no wildcards)\n"); 6149 #endif 6150 /* ANDROID CHANGES END */ 6151 ERROR("-noI\t\t\tdo not compress inode table\n"); 6152 ERROR("-noD\t\t\tdo not compress data blocks\n"); 6153 ERROR("-noF\t\t\tdo not compress fragment blocks\n"); 6154 ERROR("-noX\t\t\tdo not compress extended " 6155 "attributes\n"); 6156 ERROR("-no-fragments\t\tdo not use fragments\n"); 6157 ERROR("-always-use-fragments\tuse fragment blocks for " 6158 "files larger than block size\n"); 6159 ERROR("-no-duplicates\t\tdo not perform duplicate " 6160 "checking\n"); 6161 ERROR("-all-root\t\tmake all files owned by root\n"); 6162 ERROR("-force-uid uid\t\tset all file uids to uid\n"); 6163 ERROR("-force-gid gid\t\tset all file gids to gid\n"); 6164 ERROR("-nopad\t\t\tdo not pad filesystem to a multiple " 6165 "of 4K\n"); 6166 ERROR("-keep-as-directory\tif one source directory is " 6167 "specified, create a root\n"); 6168 ERROR("\t\t\tdirectory containing that directory, " 6169 "rather than the\n"); 6170 ERROR("\t\t\tcontents of the directory\n"); 6171 /* ANDROID CHANGES START*/ 6172 #ifdef ANDROID 6173 ERROR("-android-fs-config\tuse android fs config " 6174 "for mode, uid, and gids of inodes\n"); 6175 ERROR("-mount-point <name>\tNeed to be provided when " 6176 "android-fs-config or context-file\n\t\t\tare " 6177 "enabled and source directory is not mount point\n"); 6178 ERROR("-product-out <path>\tPRODUCT_OUT directory to " 6179 "read device specific FS rules files from\n"); 6180 ERROR("-disable-4k-align \tDon't 4k align data blocks. Default is false\n"); 6181 ERROR("-block-map <path>\tGenerate a block map for non-fragment files\n"); 6182 #endif 6183 /* ANDROID CHANGES END */ 6184 ERROR("\nFilesystem filter options:\n"); 6185 ERROR("-p <pseudo-definition>\tAdd pseudo file " 6186 "definition\n"); 6187 ERROR("-pf <pseudo-file>\tAdd list of pseudo file " 6188 "definitions\n"); 6189 ERROR("-sort <sort_file>\tsort files according to " 6190 "priorities in <sort_file>. One\n"); 6191 ERROR("\t\t\tfile or dir with priority per line. " 6192 "Priority -32768 to\n"); 6193 ERROR("\t\t\t32767, default priority 0\n"); 6194 ERROR("-ef <exclude_file>\tlist of exclude dirs/files." 6195 " One per line\n"); 6196 ERROR("-wildcards\t\tAllow extended shell wildcards " 6197 "(globbing) to be used in\n\t\t\texclude " 6198 "dirs/files\n"); 6199 ERROR("-regex\t\t\tAllow POSIX regular expressions to " 6200 "be used in exclude\n\t\t\tdirs/files\n"); 6201 ERROR("-uid-map <mapping>\tUser ID mapping.\n"); 6202 ERROR("\t\t\tFollows the format described in " 6203 "user_namespaces(7).\n"); 6204 ERROR("-gid-map <mapping>\tGroup ID mapping.\n"); 6205 ERROR("\t\t\tFollows the format described in " 6206 "user_namespaces(7).\n"); 6207 ERROR("\nFilesystem append options:\n"); 6208 ERROR("-noappend\t\tdo not append to existing " 6209 "filesystem\n"); 6210 ERROR("-root-becomes <name>\twhen appending source " 6211 "files/directories, make the\n"); 6212 ERROR("\t\t\toriginal root become a subdirectory in " 6213 "the new root\n"); 6214 ERROR("\t\t\tcalled <name>, rather than adding the new " 6215 "source items\n"); 6216 ERROR("\t\t\tto the original root\n"); 6217 ERROR("\nMksquashfs runtime options:\n"); 6218 ERROR("-version\t\tprint version, licence and " 6219 "copyright message\n"); 6220 ERROR("-exit-on-error\t\ttreat normally ignored errors " 6221 "as fatal\n"); 6222 ERROR("-recover <name>\t\trecover filesystem data " 6223 "using recovery file <name>\n"); 6224 ERROR("-no-recovery\t\tdon't generate a recovery " 6225 "file\n"); 6226 ERROR("-info\t\t\tprint files written to filesystem\n"); 6227 ERROR("-no-progress\t\tdon't display the progress " 6228 "bar\n"); 6229 ERROR("-progress\t\tdisplay progress bar when using " 6230 "the -info option\n"); 6231 ERROR("-processors <number>\tUse <number> processors." 6232 " By default will use number of\n"); 6233 ERROR("\t\t\tprocessors available\n"); 6234 ERROR("-mem <size>\t\tUse <size> physical memory. " 6235 "Currently set to %dM\n", total_mem); 6236 ERROR("\t\t\tOptionally a suffix of K, M or G can be" 6237 " given to specify\n\t\t\tKbytes, Mbytes or" 6238 " Gbytes respectively\n"); 6239 ERROR("\nMiscellaneous options:\n"); 6240 ERROR("-root-owned\t\talternative name for -all-root" 6241 "\n"); 6242 ERROR("-noInodeCompression\talternative name for -noI" 6243 "\n"); 6244 ERROR("-noDataCompression\talternative name for -noD" 6245 "\n"); 6246 ERROR("-noFragmentCompression\talternative name for " 6247 "-noF\n"); 6248 ERROR("-noXattrCompression\talternative name for " 6249 "-noX\n"); 6250 ERROR("\n-Xhelp\t\t\tprint compressor options for" 6251 " selected compressor\n"); 6252 ERROR("\nCompressors available and compressor specific " 6253 "options:\n"); 6254 display_compressor_usage(COMP_DEFAULT); 6255 exit(1); 6256 } 6257 } 6258 6259 if (!uid_map_count) { 6260 uid_mapping[0].child_id = 0; 6261 uid_mapping[0].parent_id = 0; 6262 uid_mapping[0].length = 4294967295u; 6263 uid_map_count = 1; 6264 } 6265 if (!gid_map_count) { 6266 gid_mapping[0].child_id = 0; 6267 gid_mapping[0].parent_id = 0; 6268 gid_mapping[0].length = 4294967295u; 6269 gid_map_count = 1; 6270 } 6271 6272 /* ANDROID CHANGES START*/ 6273 #ifdef ANDROID 6274 if (fs_config_file) { 6275 if (load_canned_fs_config(fs_config_file) < 0) { 6276 fprintf(stderr, "failed to load %s\n", fs_config_file); 6277 exit(1); 6278 } 6279 fs_config_func = canned_fs_config; 6280 } else if (mount_point) { 6281 fs_config_func = fs_config; 6282 } 6283 if (whitelist_filename) 6284 process_whitelist_file(whitelist_filename); 6285 #endif 6286 /* ANDROID CHANGES END */ 6287 6288 /* 6289 * Some compressors may need the options to be checked for validity 6290 * once all the options have been processed 6291 */ 6292 res = compressor_options_post(comp, block_size); 6293 if(res) 6294 EXIT_MKSQUASHFS(); 6295 6296 /* 6297 * If the -info option has been selected then disable the 6298 * progress bar unless it has been explicitly enabled with 6299 * the -progress option 6300 */ 6301 if(!silent) 6302 progress = force_progress; 6303 6304 #ifdef SQUASHFS_TRACE 6305 /* 6306 * Disable progress bar if full debug tracing is enabled. 6307 * The progress bar in this case just gets in the way of the 6308 * debug trace output 6309 */ 6310 progress = FALSE; 6311 #endif 6312 6313 for(i = 0; i < source; i++) 6314 if(lstat(source_path[i], &source_buf) == -1) { 6315 fprintf(stderr, "Cannot stat source directory \"%s\" " 6316 "because %s\n", source_path[i], 6317 strerror(errno)); 6318 EXIT_MKSQUASHFS(); 6319 } 6320 6321 destination_file = argv[source + 1]; 6322 if(stat(argv[source + 1], &buf) == -1) { 6323 if(errno == ENOENT) { /* Does not exist */ 6324 fd = open(argv[source + 1], O_CREAT | O_TRUNC | O_RDWR, 6325 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); 6326 if(fd == -1) { 6327 perror("Could not create destination file"); 6328 exit(1); 6329 } 6330 delete = TRUE; 6331 } else { 6332 perror("Could not stat destination file"); 6333 exit(1); 6334 } 6335 6336 } else { 6337 if(S_ISBLK(buf.st_mode)) { 6338 if((fd = open(argv[source + 1], O_RDWR)) == -1) { 6339 perror("Could not open block device as " 6340 "destination"); 6341 exit(1); 6342 } 6343 block_device = 1; 6344 6345 } else if(S_ISREG(buf.st_mode)) { 6346 fd = open(argv[source + 1], (delete ? O_TRUNC : 0) | 6347 O_RDWR); 6348 if(fd == -1) { 6349 perror("Could not open regular file for " 6350 "writing as destination"); 6351 exit(1); 6352 } 6353 } 6354 else { 6355 ERROR("Destination not block device or regular file\n"); 6356 exit(1); 6357 } 6358 6359 } 6360 6361 /* 6362 * process the exclude files - must be done afer destination file has 6363 * been possibly created 6364 */ 6365 for(i = source + 2; i < argc; i++) 6366 if(strcmp(argv[i], "-ef") == 0) 6367 /* 6368 * Note presence of filename arg has already 6369 * been checked 6370 */ 6371 process_exclude_file(argv[++i]); 6372 else if(strcmp(argv[i], "-e") == 0) 6373 break; 6374 else if(strcmp(argv[i], "-root-becomes") == 0 || 6375 strcmp(argv[i], "-sort") == 0 || 6376 strcmp(argv[i], "-pf") == 0 || 6377 strcmp(argv[i], "-af") == 0 || 6378 strcmp(argv[i], "-vaf") == 0 || 6379 strcmp(argv[i], "-comp") == 0) 6380 i++; 6381 6382 if(i != argc) { 6383 if(++i == argc) { 6384 ERROR("%s: -e missing arguments\n", argv[0]); 6385 EXIT_MKSQUASHFS(); 6386 } 6387 while(i < argc) 6388 if(old_exclude) 6389 old_add_exclude(argv[i++]); 6390 else 6391 add_exclude(argv[i++]); 6392 } 6393 6394 /* process the sort files - must be done afer the exclude files */ 6395 for(i = source + 2; i < argc; i++) 6396 if(strcmp(argv[i], "-sort") == 0) { 6397 int res = read_sort_file(argv[++i], source, 6398 source_path); 6399 if(res == FALSE) 6400 BAD_ERROR("Failed to read sort file\n"); 6401 sorted ++; 6402 } else if(strcmp(argv[i], "-e") == 0) 6403 break; 6404 else if(strcmp(argv[i], "-root-becomes") == 0 || 6405 strcmp(argv[i], "-ef") == 0 || 6406 strcmp(argv[i], "-pf") == 0 || 6407 strcmp(argv[i], "-af") == 0 || 6408 strcmp(argv[i], "-vaf") == 0 || 6409 strcmp(argv[i], "-comp") == 0) 6410 i++; 6411 6412 if(!delete) { 6413 comp = read_super(fd, &sBlk, argv[source + 1]); 6414 if(comp == NULL) { 6415 ERROR("Failed to read existing filesystem - will not " 6416 "overwrite - ABORTING!\n"); 6417 ERROR("To force Mksquashfs to write to this block " 6418 "device or file use -noappend\n"); 6419 EXIT_MKSQUASHFS(); 6420 } 6421 6422 block_log = slog(block_size = sBlk.block_size); 6423 noI = SQUASHFS_UNCOMPRESSED_INODES(sBlk.flags); 6424 noD = SQUASHFS_UNCOMPRESSED_DATA(sBlk.flags); 6425 noF = SQUASHFS_UNCOMPRESSED_FRAGMENTS(sBlk.flags); 6426 noX = SQUASHFS_UNCOMPRESSED_XATTRS(sBlk.flags); 6427 no_fragments = SQUASHFS_NO_FRAGMENTS(sBlk.flags); 6428 always_use_fragments = SQUASHFS_ALWAYS_FRAGMENTS(sBlk.flags); 6429 duplicate_checking = SQUASHFS_DUPLICATES(sBlk.flags); 6430 exportable = SQUASHFS_EXPORTABLE(sBlk.flags); 6431 no_xattrs = SQUASHFS_NO_XATTRS(sBlk.flags); 6432 comp_opts = SQUASHFS_COMP_OPTS(sBlk.flags); 6433 } 6434 6435 initialise_threads(readq, fragq, bwriteq, fwriteq, delete, 6436 destination_file); 6437 6438 res = compressor_init(comp, &stream, SQUASHFS_METADATA_SIZE, 0); 6439 if(res) 6440 BAD_ERROR("compressor_init failed\n"); 6441 6442 if(delete) { 6443 int size; 6444 void *comp_data = compressor_dump_options(comp, block_size, 6445 &size); 6446 6447 printf("Creating %d.%d filesystem on %s, block size %d.\n", 6448 SQUASHFS_MAJOR, SQUASHFS_MINOR, argv[source + 1], block_size); 6449 6450 /* 6451 * store any compressor specific options after the superblock, 6452 * and set the COMP_OPT flag to show that the filesystem has 6453 * compressor specfic options 6454 */ 6455 if(comp_data) { 6456 unsigned short c_byte = size | SQUASHFS_COMPRESSED_BIT; 6457 6458 SQUASHFS_INSWAP_SHORTS(&c_byte, 1); 6459 write_destination(fd, sizeof(struct squashfs_super_block), 6460 sizeof(c_byte), &c_byte); 6461 write_destination(fd, sizeof(struct squashfs_super_block) + 6462 sizeof(c_byte), size, comp_data); 6463 bytes = sizeof(struct squashfs_super_block) + sizeof(c_byte) 6464 + size; 6465 comp_opts = TRUE; 6466 } else 6467 bytes = sizeof(struct squashfs_super_block); 6468 } else { 6469 unsigned int last_directory_block, inode_dir_offset, 6470 inode_dir_file_size, root_inode_size, 6471 inode_dir_start_block, uncompressed_data, 6472 compressed_data, inode_dir_inode_number, 6473 inode_dir_parent_inode; 6474 unsigned int root_inode_start = 6475 SQUASHFS_INODE_BLK(sBlk.root_inode), 6476 root_inode_offset = 6477 SQUASHFS_INODE_OFFSET(sBlk.root_inode); 6478 6479 if((bytes = read_filesystem(root_name, fd, &sBlk, &inode_table, 6480 &data_cache, &directory_table, 6481 &directory_data_cache, &last_directory_block, 6482 &inode_dir_offset, &inode_dir_file_size, 6483 &root_inode_size, &inode_dir_start_block, 6484 &file_count, &sym_count, &dev_count, &dir_count, 6485 &fifo_count, &sock_count, &total_bytes, 6486 &total_inode_bytes, &total_directory_bytes, 6487 &inode_dir_inode_number, 6488 &inode_dir_parent_inode, add_old_root_entry, 6489 &fragment_table, &inode_lookup_table)) == 0) { 6490 ERROR("Failed to read existing filesystem - will not " 6491 "overwrite - ABORTING!\n"); 6492 ERROR("To force Mksquashfs to write to this block " 6493 "device or file use -noappend\n"); 6494 EXIT_MKSQUASHFS(); 6495 } 6496 if((append_fragments = fragments = sBlk.fragments)) { 6497 fragment_table = realloc((char *) fragment_table, 6498 ((fragments + FRAG_SIZE - 1) & ~(FRAG_SIZE - 1)) 6499 * sizeof(struct squashfs_fragment_entry)); 6500 if(fragment_table == NULL) 6501 BAD_ERROR("Out of memory in save filesystem state\n"); 6502 } 6503 6504 printf("Appending to existing %d.%d filesystem on %s, block " 6505 "size %d\n", SQUASHFS_MAJOR, SQUASHFS_MINOR, argv[source + 1], 6506 block_size); 6507 printf("All -b, -noI, -noD, -noF, -noX, no-duplicates, no-fragments, " 6508 "-always-use-fragments,\n-exportable and -comp options " 6509 "ignored\n"); 6510 printf("\nIf appending is not wanted, please re-run with " 6511 "-noappend specified!\n\n"); 6512 6513 compressed_data = (inode_dir_offset + inode_dir_file_size) & 6514 ~(SQUASHFS_METADATA_SIZE - 1); 6515 uncompressed_data = (inode_dir_offset + inode_dir_file_size) & 6516 (SQUASHFS_METADATA_SIZE - 1); 6517 6518 /* save original filesystem state for restoring ... */ 6519 sfragments = fragments; 6520 sbytes = bytes; 6521 sinode_count = sBlk.inodes; 6522 scache_bytes = root_inode_offset + root_inode_size; 6523 sdirectory_cache_bytes = uncompressed_data; 6524 sdata_cache = malloc(scache_bytes); 6525 if(sdata_cache == NULL) 6526 BAD_ERROR("Out of memory in save filesystem state\n"); 6527 sdirectory_data_cache = malloc(sdirectory_cache_bytes); 6528 if(sdirectory_data_cache == NULL) 6529 BAD_ERROR("Out of memory in save filesystem state\n"); 6530 memcpy(sdata_cache, data_cache, scache_bytes); 6531 memcpy(sdirectory_data_cache, directory_data_cache + 6532 compressed_data, sdirectory_cache_bytes); 6533 sinode_bytes = root_inode_start; 6534 stotal_bytes = total_bytes; 6535 stotal_inode_bytes = total_inode_bytes; 6536 stotal_directory_bytes = total_directory_bytes + 6537 compressed_data; 6538 sfile_count = file_count; 6539 ssym_count = sym_count; 6540 sdev_count = dev_count; 6541 sdir_count = dir_count + 1; 6542 sfifo_count = fifo_count; 6543 ssock_count = sock_count; 6544 sdup_files = dup_files; 6545 sid_count = id_count; 6546 write_recovery_data(&sBlk); 6547 save_xattrs(); 6548 appending = TRUE; 6549 6550 /* 6551 * set the filesystem state up to be able to append to the 6552 * original filesystem. The filesystem state differs depending 6553 * on whether we're appending to the original root directory, or 6554 * if the original root directory becomes a sub-directory 6555 * (root-becomes specified on command line, here root_name != 6556 * NULL) 6557 */ 6558 inode_bytes = inode_size = root_inode_start; 6559 directory_size = last_directory_block; 6560 cache_size = root_inode_offset + root_inode_size; 6561 directory_cache_size = inode_dir_offset + inode_dir_file_size; 6562 if(root_name) { 6563 sdirectory_bytes = last_directory_block; 6564 sdirectory_compressed_bytes = 0; 6565 root_inode_number = inode_dir_parent_inode; 6566 inode_no = sBlk.inodes + 2; 6567 directory_bytes = last_directory_block; 6568 directory_cache_bytes = uncompressed_data; 6569 memmove(directory_data_cache, directory_data_cache + 6570 compressed_data, uncompressed_data); 6571 cache_bytes = root_inode_offset + root_inode_size; 6572 add_old_root_entry(root_name, sBlk.root_inode, 6573 inode_dir_inode_number, SQUASHFS_DIR_TYPE); 6574 total_directory_bytes += compressed_data; 6575 dir_count ++; 6576 } else { 6577 sdirectory_compressed_bytes = last_directory_block - 6578 inode_dir_start_block; 6579 sdirectory_compressed = 6580 malloc(sdirectory_compressed_bytes); 6581 if(sdirectory_compressed == NULL) 6582 BAD_ERROR("Out of memory in save filesystem " 6583 "state\n"); 6584 memcpy(sdirectory_compressed, directory_table + 6585 inode_dir_start_block, 6586 sdirectory_compressed_bytes); 6587 sdirectory_bytes = inode_dir_start_block; 6588 root_inode_number = inode_dir_inode_number; 6589 inode_no = sBlk.inodes + 1; 6590 directory_bytes = inode_dir_start_block; 6591 directory_cache_bytes = inode_dir_offset; 6592 cache_bytes = root_inode_offset; 6593 } 6594 6595 inode_count = file_count + dir_count + sym_count + dev_count + 6596 fifo_count + sock_count; 6597 } 6598 6599 if(path) 6600 paths = add_subdir(paths, path); 6601 6602 dump_actions(); 6603 dump_pseudos(); 6604 6605 if(delete && !keep_as_directory && source == 1 && 6606 S_ISDIR(source_buf.st_mode)) 6607 dir_scan(&inode, source_path[0], scan1_readdir, progress); 6608 else if(!keep_as_directory && source == 1 && 6609 S_ISDIR(source_buf.st_mode)) 6610 dir_scan(&inode, source_path[0], scan1_single_readdir, progress); 6611 else 6612 dir_scan(&inode, "", scan1_encomp_readdir, progress); 6613 sBlk.root_inode = inode; 6614 sBlk.inodes = inode_count; 6615 sBlk.s_magic = SQUASHFS_MAGIC; 6616 sBlk.s_major = SQUASHFS_MAJOR; 6617 sBlk.s_minor = SQUASHFS_MINOR; 6618 sBlk.block_size = block_size; 6619 sBlk.block_log = block_log; 6620 sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments, 6621 always_use_fragments, duplicate_checking, exportable, 6622 no_xattrs, comp_opts); 6623 sBlk.mkfs_time = time(NULL); 6624 6625 disable_info(); 6626 6627 while((fragment = get_frag_action(fragment))) 6628 write_fragment(*fragment); 6629 unlock_fragments(); 6630 pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 6631 pthread_mutex_lock(&fragment_mutex); 6632 while(fragments_outstanding) { 6633 pthread_mutex_unlock(&fragment_mutex); 6634 sched_yield(); 6635 pthread_mutex_lock(&fragment_mutex); 6636 } 6637 pthread_cleanup_pop(1); 6638 6639 queue_put(to_writer, NULL); 6640 if(queue_get(from_writer) != 0) 6641 EXIT_MKSQUASHFS(); 6642 6643 set_progressbar_state(FALSE); 6644 write_filesystem_tables(&sBlk, nopad); 6645 6646 /* ANDROID CHANGES START*/ 6647 #ifdef ANDROID 6648 if (block_map_file) 6649 fclose(block_map_file); 6650 #endif 6651 /* ANDROID CHANGES END */ 6652 6653 return 0; 6654 } 6655