1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #define _FILE_OFFSET_BITS 64 18 #define _LARGEFILE64_SOURCE 1 19 20 #include <fcntl.h> 21 #include <limits.h> 22 #include <stdbool.h> 23 #include <stddef.h> 24 #include <stdlib.h> 25 #include <string.h> 26 #include <sys/stat.h> 27 #include <sys/types.h> 28 #include <unistd.h> 29 #include <zlib.h> 30 31 #include "output_file.h" 32 #include "sparse_format.h" 33 #include "sparse_crc32.h" 34 35 #ifndef USE_MINGW 36 #include <sys/mman.h> 37 #define O_BINARY 0 38 #else 39 #define ftruncate64 ftruncate 40 #endif 41 42 #if defined(__APPLE__) && defined(__MACH__) 43 #define lseek64 lseek 44 #define ftruncate64 ftruncate 45 #define mmap64 mmap 46 #define off64_t off_t 47 #endif 48 49 #ifdef __BIONIC__ 50 extern void* __mmap2(void *, size_t, int, int, int, off_t); 51 static inline void *mmap64(void *addr, size_t length, int prot, int flags, 52 int fd, off64_t offset) 53 { 54 return __mmap2(addr, length, prot, flags, fd, offset >> 12); 55 } 56 #endif 57 58 #define min(a, b) \ 59 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) 60 61 #define SPARSE_HEADER_MAJOR_VER 1 62 #define SPARSE_HEADER_MINOR_VER 0 63 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 64 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 65 66 #define container_of(inner, outer_t, elem) \ 67 ((outer_t *)((char *)inner - offsetof(outer_t, elem))) 68 69 struct output_file_ops { 70 int (*open)(struct output_file *, int fd); 71 int (*skip)(struct output_file *, int64_t); 72 int (*pad)(struct output_file *, int64_t); 73 int (*write)(struct output_file *, void *, int); 74 void (*close)(struct output_file *); 75 }; 76 77 struct sparse_file_ops { 78 int (*write_data_chunk)(struct output_file *out, unsigned int len, 79 void *data); 80 int (*write_fill_chunk)(struct output_file *out, unsigned int len, 81 uint32_t fill_val); 82 int (*write_skip_chunk)(struct output_file *out, int64_t len); 83 int (*write_end_chunk)(struct output_file *out); 84 }; 85 86 struct output_file { 87 int64_t cur_out_ptr; 88 unsigned int chunk_cnt; 89 uint32_t crc32; 90 struct output_file_ops *ops; 91 struct sparse_file_ops *sparse_ops; 92 int use_crc; 93 unsigned int block_size; 94 int64_t len; 95 char *zero_buf; 96 uint32_t *fill_buf; 97 char *buf; 98 }; 99 100 struct output_file_gz { 101 struct output_file out; 102 gzFile gz_fd; 103 }; 104 105 #define to_output_file_gz(_o) \ 106 container_of((_o), struct output_file_gz, out) 107 108 struct output_file_normal { 109 struct output_file out; 110 int fd; 111 }; 112 113 #define to_output_file_normal(_o) \ 114 container_of((_o), struct output_file_normal, out) 115 116 struct output_file_callback { 117 struct output_file out; 118 void *priv; 119 int (*write)(void *priv, const void *buf, int len); 120 }; 121 122 #define to_output_file_callback(_o) \ 123 container_of((_o), struct output_file_callback, out) 124 125 static int file_open(struct output_file *out, int fd) 126 { 127 struct output_file_normal *outn = to_output_file_normal(out); 128 129 outn->fd = fd; 130 return 0; 131 } 132 133 static int file_skip(struct output_file *out, int64_t cnt) 134 { 135 off64_t ret; 136 struct output_file_normal *outn = to_output_file_normal(out); 137 138 ret = lseek64(outn->fd, cnt, SEEK_CUR); 139 if (ret < 0) { 140 error_errno("lseek64"); 141 return -1; 142 } 143 return 0; 144 } 145 146 static int file_pad(struct output_file *out, int64_t len) 147 { 148 int ret; 149 struct output_file_normal *outn = to_output_file_normal(out); 150 151 ret = ftruncate64(outn->fd, len); 152 if (ret < 0) { 153 return -errno; 154 } 155 156 return 0; 157 } 158 159 static int file_write(struct output_file *out, void *data, int len) 160 { 161 int ret; 162 struct output_file_normal *outn = to_output_file_normal(out); 163 164 ret = write(outn->fd, data, len); 165 if (ret < 0) { 166 error_errno("write"); 167 return -1; 168 } else if (ret < len) { 169 error("incomplete write"); 170 return -1; 171 } 172 173 return 0; 174 } 175 176 static void file_close(struct output_file *out) 177 { 178 struct output_file_normal *outn = to_output_file_normal(out); 179 180 free(outn); 181 } 182 183 static struct output_file_ops file_ops = { 184 .open = file_open, 185 .skip = file_skip, 186 .pad = file_pad, 187 .write = file_write, 188 .close = file_close, 189 }; 190 191 static int gz_file_open(struct output_file *out, int fd) 192 { 193 struct output_file_gz *outgz = to_output_file_gz(out); 194 195 outgz->gz_fd = gzdopen(fd, "wb9"); 196 if (!outgz->gz_fd) { 197 error_errno("gzopen"); 198 return -errno; 199 } 200 201 return 0; 202 } 203 204 205 static int gz_file_skip(struct output_file *out, int64_t cnt) 206 { 207 off64_t ret; 208 struct output_file_gz *outgz = to_output_file_gz(out); 209 210 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); 211 if (ret < 0) { 212 error_errno("gzseek"); 213 return -1; 214 } 215 return 0; 216 } 217 218 static int gz_file_pad(struct output_file *out, int64_t len) 219 { 220 off64_t ret; 221 struct output_file_gz *outgz = to_output_file_gz(out); 222 223 ret = gztell(outgz->gz_fd); 224 if (ret < 0) { 225 return -1; 226 } 227 228 if (ret >= len) { 229 return 0; 230 } 231 232 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); 233 if (ret < 0) { 234 return -1; 235 } 236 237 gzwrite(outgz->gz_fd, "", 1); 238 239 return 0; 240 } 241 242 static int gz_file_write(struct output_file *out, void *data, int len) 243 { 244 int ret; 245 struct output_file_gz *outgz = to_output_file_gz(out); 246 247 ret = gzwrite(outgz->gz_fd, data, len); 248 if (ret < 0) { 249 error_errno("gzwrite"); 250 return -1; 251 } else if (ret < len) { 252 error("incomplete gzwrite"); 253 return -1; 254 } 255 256 return 0; 257 } 258 259 static void gz_file_close(struct output_file *out) 260 { 261 struct output_file_gz *outgz = to_output_file_gz(out); 262 263 gzclose(outgz->gz_fd); 264 free(outgz); 265 } 266 267 static struct output_file_ops gz_file_ops = { 268 .open = gz_file_open, 269 .skip = gz_file_skip, 270 .pad = gz_file_pad, 271 .write = gz_file_write, 272 .close = gz_file_close, 273 }; 274 275 static int callback_file_open(struct output_file *out, int fd) 276 { 277 return 0; 278 } 279 280 static int callback_file_skip(struct output_file *out, int64_t off) 281 { 282 struct output_file_callback *outc = to_output_file_callback(out); 283 int to_write; 284 int ret; 285 286 while (off > 0) { 287 to_write = min(off, (int64_t)INT_MAX); 288 ret = outc->write(outc->priv, NULL, to_write); 289 if (ret < 0) { 290 return ret; 291 } 292 off -= to_write; 293 } 294 295 return 0; 296 } 297 298 static int callback_file_pad(struct output_file *out, int64_t len) 299 { 300 return -1; 301 } 302 303 static int callback_file_write(struct output_file *out, void *data, int len) 304 { 305 int ret; 306 struct output_file_callback *outc = to_output_file_callback(out); 307 308 return outc->write(outc->priv, data, len); 309 } 310 311 static void callback_file_close(struct output_file *out) 312 { 313 struct output_file_callback *outc = to_output_file_callback(out); 314 315 free(outc); 316 } 317 318 static struct output_file_ops callback_file_ops = { 319 .open = callback_file_open, 320 .skip = callback_file_skip, 321 .pad = callback_file_pad, 322 .write = callback_file_write, 323 .close = callback_file_close, 324 }; 325 326 int read_all(int fd, void *buf, size_t len) 327 { 328 size_t total = 0; 329 int ret; 330 char *ptr = buf; 331 332 while (total < len) { 333 ret = read(fd, ptr, len - total); 334 335 if (ret < 0) 336 return -errno; 337 338 if (ret == 0) 339 return -EINVAL; 340 341 ptr += ret; 342 total += ret; 343 } 344 345 return 0; 346 } 347 348 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) 349 { 350 chunk_header_t chunk_header; 351 int ret, chunk; 352 353 if (skip_len % out->block_size) { 354 error("don't care size %llu is not a multiple of the block size %u", 355 skip_len, out->block_size); 356 return -1; 357 } 358 359 /* We are skipping data, so emit a don't care chunk. */ 360 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 361 chunk_header.reserved1 = 0; 362 chunk_header.chunk_sz = skip_len / out->block_size; 363 chunk_header.total_sz = CHUNK_HEADER_LEN; 364 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 365 if (ret < 0) 366 return -1; 367 368 out->cur_out_ptr += skip_len; 369 out->chunk_cnt++; 370 371 return 0; 372 } 373 374 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, 375 uint32_t fill_val) 376 { 377 chunk_header_t chunk_header; 378 int rnd_up_len, zero_len, count; 379 int ret; 380 unsigned int i; 381 382 /* Round up the fill length to a multiple of the block size */ 383 rnd_up_len = ALIGN(len, out->block_size); 384 385 /* Finally we can safely emit a chunk of data */ 386 chunk_header.chunk_type = CHUNK_TYPE_FILL; 387 chunk_header.reserved1 = 0; 388 chunk_header.chunk_sz = rnd_up_len / out->block_size; 389 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 390 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 391 392 if (ret < 0) 393 return -1; 394 ret = out->ops->write(out, &fill_val, sizeof(fill_val)); 395 if (ret < 0) 396 return -1; 397 398 if (out->use_crc) { 399 count = out->block_size / sizeof(uint32_t); 400 while (count--) 401 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); 402 } 403 404 out->cur_out_ptr += rnd_up_len; 405 out->chunk_cnt++; 406 407 return 0; 408 } 409 410 static int write_sparse_data_chunk(struct output_file *out, unsigned int len, 411 void *data) 412 { 413 chunk_header_t chunk_header; 414 int rnd_up_len, zero_len; 415 int ret; 416 417 /* Round up the data length to a multiple of the block size */ 418 rnd_up_len = ALIGN(len, out->block_size); 419 zero_len = rnd_up_len - len; 420 421 /* Finally we can safely emit a chunk of data */ 422 chunk_header.chunk_type = CHUNK_TYPE_RAW; 423 chunk_header.reserved1 = 0; 424 chunk_header.chunk_sz = rnd_up_len / out->block_size; 425 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 426 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 427 428 if (ret < 0) 429 return -1; 430 ret = out->ops->write(out, data, len); 431 if (ret < 0) 432 return -1; 433 if (zero_len) { 434 ret = out->ops->write(out, out->zero_buf, zero_len); 435 if (ret < 0) 436 return -1; 437 } 438 439 if (out->use_crc) { 440 out->crc32 = sparse_crc32(out->crc32, data, len); 441 if (zero_len) 442 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); 443 } 444 445 out->cur_out_ptr += rnd_up_len; 446 out->chunk_cnt++; 447 448 return 0; 449 } 450 451 int write_sparse_end_chunk(struct output_file *out) 452 { 453 chunk_header_t chunk_header; 454 int ret; 455 456 if (out->use_crc) { 457 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 458 chunk_header.reserved1 = 0; 459 chunk_header.chunk_sz = 0; 460 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 461 462 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 463 if (ret < 0) { 464 return ret; 465 } 466 out->ops->write(out, &out->crc32, 4); 467 if (ret < 0) { 468 return ret; 469 } 470 471 out->chunk_cnt++; 472 } 473 474 return 0; 475 } 476 477 static struct sparse_file_ops sparse_file_ops = { 478 .write_data_chunk = write_sparse_data_chunk, 479 .write_fill_chunk = write_sparse_fill_chunk, 480 .write_skip_chunk = write_sparse_skip_chunk, 481 .write_end_chunk = write_sparse_end_chunk, 482 }; 483 484 static int write_normal_data_chunk(struct output_file *out, unsigned int len, 485 void *data) 486 { 487 int ret; 488 unsigned int rnd_up_len = ALIGN(len, out->block_size); 489 490 ret = out->ops->write(out, data, len); 491 if (ret < 0) { 492 return ret; 493 } 494 495 if (rnd_up_len > len) { 496 ret = out->ops->skip(out, rnd_up_len - len); 497 } 498 499 return ret; 500 } 501 502 static int write_normal_fill_chunk(struct output_file *out, unsigned int len, 503 uint32_t fill_val) 504 { 505 int ret; 506 unsigned int i; 507 unsigned int write_len; 508 509 /* Initialize fill_buf with the fill_val */ 510 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { 511 out->fill_buf[i] = fill_val; 512 } 513 514 while (len) { 515 write_len = min(len, out->block_size); 516 ret = out->ops->write(out, out->fill_buf, write_len); 517 if (ret < 0) { 518 return ret; 519 } 520 521 len -= write_len; 522 } 523 524 return 0; 525 } 526 527 static int write_normal_skip_chunk(struct output_file *out, int64_t len) 528 { 529 return out->ops->skip(out, len); 530 } 531 532 int write_normal_end_chunk(struct output_file *out) 533 { 534 return out->ops->pad(out, out->len); 535 } 536 537 static struct sparse_file_ops normal_file_ops = { 538 .write_data_chunk = write_normal_data_chunk, 539 .write_fill_chunk = write_normal_fill_chunk, 540 .write_skip_chunk = write_normal_skip_chunk, 541 .write_end_chunk = write_normal_end_chunk, 542 }; 543 544 void output_file_close(struct output_file *out) 545 { 546 int ret; 547 548 out->sparse_ops->write_end_chunk(out); 549 out->ops->close(out); 550 } 551 552 static int output_file_init(struct output_file *out, int block_size, 553 int64_t len, bool sparse, int chunks, bool crc) 554 { 555 int ret; 556 557 out->len = len; 558 out->block_size = block_size; 559 out->cur_out_ptr = 0ll; 560 out->chunk_cnt = 0; 561 out->crc32 = 0; 562 out->use_crc = crc; 563 564 out->zero_buf = calloc(block_size, 1); 565 if (!out->zero_buf) { 566 error_errno("malloc zero_buf"); 567 return -ENOMEM; 568 } 569 570 out->fill_buf = calloc(block_size, 1); 571 if (!out->fill_buf) { 572 error_errno("malloc fill_buf"); 573 ret = -ENOMEM; 574 goto err_fill_buf; 575 } 576 577 if (sparse) { 578 out->sparse_ops = &sparse_file_ops; 579 } else { 580 out->sparse_ops = &normal_file_ops; 581 } 582 583 if (sparse) { 584 sparse_header_t sparse_header = { 585 .magic = SPARSE_HEADER_MAGIC, 586 .major_version = SPARSE_HEADER_MAJOR_VER, 587 .minor_version = SPARSE_HEADER_MINOR_VER, 588 .file_hdr_sz = SPARSE_HEADER_LEN, 589 .chunk_hdr_sz = CHUNK_HEADER_LEN, 590 .blk_sz = out->block_size, 591 .total_blks = out->len / out->block_size, 592 .total_chunks = chunks, 593 .image_checksum = 0 594 }; 595 596 if (out->use_crc) { 597 sparse_header.total_chunks++; 598 } 599 600 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); 601 if (ret < 0) { 602 goto err_write; 603 } 604 } 605 606 return 0; 607 608 err_write: 609 free(out->fill_buf); 610 err_fill_buf: 611 free(out->zero_buf); 612 return ret; 613 } 614 615 static struct output_file *output_file_new_gz(void) 616 { 617 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz)); 618 if (!outgz) { 619 error_errno("malloc struct outgz"); 620 return NULL; 621 } 622 623 outgz->out.ops = &gz_file_ops; 624 625 return &outgz->out; 626 } 627 628 static struct output_file *output_file_new_normal(void) 629 { 630 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal)); 631 if (!outn) { 632 error_errno("malloc struct outn"); 633 return NULL; 634 } 635 636 outn->out.ops = &file_ops; 637 638 return &outn->out; 639 } 640 641 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int), 642 void *priv, unsigned int block_size, int64_t len, int gz, int sparse, 643 int chunks, int crc) 644 { 645 int ret; 646 struct output_file_callback *outc; 647 648 outc = calloc(1, sizeof(struct output_file_callback)); 649 if (!outc) { 650 error_errno("malloc struct outc"); 651 return NULL; 652 } 653 654 outc->out.ops = &callback_file_ops; 655 outc->priv = priv; 656 outc->write = write; 657 658 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc); 659 if (ret < 0) { 660 free(outc); 661 return NULL; 662 } 663 664 return &outc->out; 665 } 666 667 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len, 668 int gz, int sparse, int chunks, int crc) 669 { 670 int ret; 671 struct output_file *out; 672 673 if (gz) { 674 out = output_file_new_gz(); 675 } else { 676 out = output_file_new_normal(); 677 } 678 if (!out) { 679 return NULL; 680 } 681 682 out->ops->open(out, fd); 683 684 ret = output_file_init(out, block_size, len, sparse, chunks, crc); 685 if (ret < 0) { 686 free(out); 687 return NULL; 688 } 689 690 return out; 691 } 692 693 /* Write a contiguous region of data blocks from a memory buffer */ 694 int write_data_chunk(struct output_file *out, unsigned int len, void *data) 695 { 696 return out->sparse_ops->write_data_chunk(out, len, data); 697 } 698 699 /* Write a contiguous region of data blocks with a fill value */ 700 int write_fill_chunk(struct output_file *out, unsigned int len, 701 uint32_t fill_val) 702 { 703 return out->sparse_ops->write_fill_chunk(out, len, fill_val); 704 } 705 706 int write_fd_chunk(struct output_file *out, unsigned int len, 707 int fd, int64_t offset) 708 { 709 int ret; 710 int64_t aligned_offset; 711 int aligned_diff; 712 int buffer_size; 713 char *ptr; 714 715 aligned_offset = offset & ~(4096 - 1); 716 aligned_diff = offset - aligned_offset; 717 buffer_size = len + aligned_diff; 718 719 #ifndef USE_MINGW 720 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, 721 aligned_offset); 722 if (data == MAP_FAILED) { 723 return -errno; 724 } 725 ptr = data + aligned_diff; 726 #else 727 off64_t pos; 728 char *data = malloc(len); 729 if (!data) { 730 return -errno; 731 } 732 pos = lseek64(fd, offset, SEEK_SET); 733 if (pos < 0) { 734 return -errno; 735 } 736 ret = read_all(fd, data, len); 737 if (ret < 0) { 738 return ret; 739 } 740 ptr = data; 741 #endif 742 743 ret = out->sparse_ops->write_data_chunk(out, len, ptr); 744 745 #ifndef USE_MINGW 746 munmap(data, buffer_size); 747 #else 748 free(data); 749 #endif 750 751 return ret; 752 } 753 754 /* Write a contiguous region of data blocks from a file */ 755 int write_file_chunk(struct output_file *out, unsigned int len, 756 const char *file, int64_t offset) 757 { 758 int ret; 759 760 int file_fd = open(file, O_RDONLY | O_BINARY); 761 if (file_fd < 0) { 762 return -errno; 763 } 764 765 ret = write_fd_chunk(out, len, file_fd, offset); 766 767 close(file_fd); 768 769 return ret; 770 } 771 772 int write_skip_chunk(struct output_file *out, int64_t len) 773 { 774 return out->sparse_ops->write_skip_chunk(out, len); 775 } 776
