1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #define _FILE_OFFSET_BITS 64 18 #define _LARGEFILE64_SOURCE 1 19 20 #include <fcntl.h> 21 #include <inttypes.h> 22 #include <limits.h> 23 #include <stdbool.h> 24 #include <stddef.h> 25 #include <stdlib.h> 26 #include <string.h> 27 #include <sys/stat.h> 28 #include <sys/types.h> 29 #include <unistd.h> 30 #include <zlib.h> 31 32 #include "defs.h" 33 #include "output_file.h" 34 #include "sparse_crc32.h" 35 #include "sparse_format.h" 36 37 #ifndef _WIN32 38 #include <sys/mman.h> 39 #define O_BINARY 0 40 #else 41 #define ftruncate64 ftruncate 42 #endif 43 44 #if defined(__APPLE__) && defined(__MACH__) 45 #define lseek64 lseek 46 #define ftruncate64 ftruncate 47 #define mmap64 mmap 48 #define off64_t off_t 49 #endif 50 51 #define min(a, b) \ 52 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; }) 53 54 #define SPARSE_HEADER_MAJOR_VER 1 55 #define SPARSE_HEADER_MINOR_VER 0 56 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 57 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 58 59 #define container_of(inner, outer_t, elem) \ 60 ((outer_t *)((char *)(inner) - offsetof(outer_t, elem))) 61 62 struct output_file_ops { 63 int (*open)(struct output_file *, int fd); 64 int (*skip)(struct output_file *, int64_t); 65 int (*pad)(struct output_file *, int64_t); 66 int (*write)(struct output_file *, void *, size_t); 67 void (*close)(struct output_file *); 68 }; 69 70 struct sparse_file_ops { 71 int (*write_data_chunk)(struct output_file *out, unsigned int len, 72 void *data); 73 int (*write_fill_chunk)(struct output_file *out, unsigned int len, 74 uint32_t fill_val); 75 int (*write_skip_chunk)(struct output_file *out, int64_t len); 76 int (*write_end_chunk)(struct output_file *out); 77 }; 78 79 struct output_file { 80 int64_t cur_out_ptr; 81 unsigned int chunk_cnt; 82 uint32_t crc32; 83 struct output_file_ops *ops; 84 struct sparse_file_ops *sparse_ops; 85 int use_crc; 86 unsigned int block_size; 87 int64_t len; 88 char *zero_buf; 89 uint32_t *fill_buf; 90 char *buf; 91 }; 92 93 struct output_file_gz { 94 struct output_file out; 95 gzFile gz_fd; 96 }; 97 98 #define to_output_file_gz(_o) \ 99 container_of((_o), struct output_file_gz, out) 100 101 struct output_file_normal { 102 struct output_file out; 103 int fd; 104 }; 105 106 #define to_output_file_normal(_o) \ 107 container_of((_o), struct output_file_normal, out) 108 109 struct output_file_callback { 110 struct output_file out; 111 void *priv; 112 int (*write)(void *priv, const void *buf, int len); 113 }; 114 115 #define to_output_file_callback(_o) \ 116 container_of((_o), struct output_file_callback, out) 117 118 static int file_open(struct output_file *out, int fd) 119 { 120 struct output_file_normal *outn = to_output_file_normal(out); 121 122 outn->fd = fd; 123 return 0; 124 } 125 126 static int file_skip(struct output_file *out, int64_t cnt) 127 { 128 off64_t ret; 129 struct output_file_normal *outn = to_output_file_normal(out); 130 131 ret = lseek64(outn->fd, cnt, SEEK_CUR); 132 if (ret < 0) { 133 error_errno("lseek64"); 134 return -1; 135 } 136 return 0; 137 } 138 139 static int file_pad(struct output_file *out, int64_t len) 140 { 141 int ret; 142 struct output_file_normal *outn = to_output_file_normal(out); 143 144 ret = ftruncate64(outn->fd, len); 145 if (ret < 0) { 146 return -errno; 147 } 148 149 return 0; 150 } 151 152 static int file_write(struct output_file *out, void *data, size_t len) 153 { 154 ssize_t ret; 155 struct output_file_normal *outn = to_output_file_normal(out); 156 157 while (len > 0) { 158 ret = write(outn->fd, data, len); 159 if (ret < 0) { 160 if (errno == EINTR) { 161 continue; 162 } 163 error_errno("write"); 164 return -1; 165 } 166 167 data = (char *)data + ret; 168 len -= ret; 169 } 170 171 return 0; 172 } 173 174 static void file_close(struct output_file *out) 175 { 176 struct output_file_normal *outn = to_output_file_normal(out); 177 178 free(outn); 179 } 180 181 static struct output_file_ops file_ops = { 182 .open = file_open, 183 .skip = file_skip, 184 .pad = file_pad, 185 .write = file_write, 186 .close = file_close, 187 }; 188 189 static int gz_file_open(struct output_file *out, int fd) 190 { 191 struct output_file_gz *outgz = to_output_file_gz(out); 192 193 outgz->gz_fd = gzdopen(fd, "wb9"); 194 if (!outgz->gz_fd) { 195 error_errno("gzopen"); 196 return -errno; 197 } 198 199 return 0; 200 } 201 202 203 static int gz_file_skip(struct output_file *out, int64_t cnt) 204 { 205 off64_t ret; 206 struct output_file_gz *outgz = to_output_file_gz(out); 207 208 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR); 209 if (ret < 0) { 210 error_errno("gzseek"); 211 return -1; 212 } 213 return 0; 214 } 215 216 static int gz_file_pad(struct output_file *out, int64_t len) 217 { 218 off64_t ret; 219 struct output_file_gz *outgz = to_output_file_gz(out); 220 221 ret = gztell(outgz->gz_fd); 222 if (ret < 0) { 223 return -1; 224 } 225 226 if (ret >= len) { 227 return 0; 228 } 229 230 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET); 231 if (ret < 0) { 232 return -1; 233 } 234 235 gzwrite(outgz->gz_fd, "", 1); 236 237 return 0; 238 } 239 240 static int gz_file_write(struct output_file *out, void *data, size_t len) 241 { 242 int ret; 243 struct output_file_gz *outgz = to_output_file_gz(out); 244 245 while (len > 0) { 246 ret = gzwrite(outgz->gz_fd, data, 247 min(len, (unsigned int)INT_MAX)); 248 if (ret == 0) { 249 error("gzwrite %s", gzerror(outgz->gz_fd, NULL)); 250 return -1; 251 } 252 len -= ret; 253 data = (char *)data + ret; 254 } 255 256 return 0; 257 } 258 259 static void gz_file_close(struct output_file *out) 260 { 261 struct output_file_gz *outgz = to_output_file_gz(out); 262 263 gzclose(outgz->gz_fd); 264 free(outgz); 265 } 266 267 static struct output_file_ops gz_file_ops = { 268 .open = gz_file_open, 269 .skip = gz_file_skip, 270 .pad = gz_file_pad, 271 .write = gz_file_write, 272 .close = gz_file_close, 273 }; 274 275 static int callback_file_open(struct output_file *out __unused, int fd __unused) 276 { 277 return 0; 278 } 279 280 static int callback_file_skip(struct output_file *out, int64_t off) 281 { 282 struct output_file_callback *outc = to_output_file_callback(out); 283 int to_write; 284 int ret; 285 286 while (off > 0) { 287 to_write = min(off, (int64_t)INT_MAX); 288 ret = outc->write(outc->priv, NULL, to_write); 289 if (ret < 0) { 290 return ret; 291 } 292 off -= to_write; 293 } 294 295 return 0; 296 } 297 298 static int callback_file_pad(struct output_file *out __unused, int64_t len __unused) 299 { 300 return -1; 301 } 302 303 static int callback_file_write(struct output_file *out, void *data, size_t len) 304 { 305 struct output_file_callback *outc = to_output_file_callback(out); 306 307 return outc->write(outc->priv, data, len); 308 } 309 310 static void callback_file_close(struct output_file *out) 311 { 312 struct output_file_callback *outc = to_output_file_callback(out); 313 314 free(outc); 315 } 316 317 static struct output_file_ops callback_file_ops = { 318 .open = callback_file_open, 319 .skip = callback_file_skip, 320 .pad = callback_file_pad, 321 .write = callback_file_write, 322 .close = callback_file_close, 323 }; 324 325 int read_all(int fd, void *buf, size_t len) 326 { 327 size_t total = 0; 328 int ret; 329 char *ptr = buf; 330 331 while (total < len) { 332 ret = read(fd, ptr, len - total); 333 334 if (ret < 0) 335 return -errno; 336 337 if (ret == 0) 338 return -EINVAL; 339 340 ptr += ret; 341 total += ret; 342 } 343 344 return 0; 345 } 346 347 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len) 348 { 349 chunk_header_t chunk_header; 350 int ret; 351 352 if (skip_len % out->block_size) { 353 error("don't care size %"PRIi64" is not a multiple of the block size %u", 354 skip_len, out->block_size); 355 return -1; 356 } 357 358 /* We are skipping data, so emit a don't care chunk. */ 359 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 360 chunk_header.reserved1 = 0; 361 chunk_header.chunk_sz = skip_len / out->block_size; 362 chunk_header.total_sz = CHUNK_HEADER_LEN; 363 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 364 if (ret < 0) 365 return -1; 366 367 out->cur_out_ptr += skip_len; 368 out->chunk_cnt++; 369 370 return 0; 371 } 372 373 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len, 374 uint32_t fill_val) 375 { 376 chunk_header_t chunk_header; 377 int rnd_up_len, count; 378 int ret; 379 380 /* Round up the fill length to a multiple of the block size */ 381 rnd_up_len = ALIGN(len, out->block_size); 382 383 /* Finally we can safely emit a chunk of data */ 384 chunk_header.chunk_type = CHUNK_TYPE_FILL; 385 chunk_header.reserved1 = 0; 386 chunk_header.chunk_sz = rnd_up_len / out->block_size; 387 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 388 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 389 390 if (ret < 0) 391 return -1; 392 ret = out->ops->write(out, &fill_val, sizeof(fill_val)); 393 if (ret < 0) 394 return -1; 395 396 if (out->use_crc) { 397 count = out->block_size / sizeof(uint32_t); 398 while (count--) 399 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t)); 400 } 401 402 out->cur_out_ptr += rnd_up_len; 403 out->chunk_cnt++; 404 405 return 0; 406 } 407 408 static int write_sparse_data_chunk(struct output_file *out, unsigned int len, 409 void *data) 410 { 411 chunk_header_t chunk_header; 412 int rnd_up_len, zero_len; 413 int ret; 414 415 /* Round up the data length to a multiple of the block size */ 416 rnd_up_len = ALIGN(len, out->block_size); 417 zero_len = rnd_up_len - len; 418 419 /* Finally we can safely emit a chunk of data */ 420 chunk_header.chunk_type = CHUNK_TYPE_RAW; 421 chunk_header.reserved1 = 0; 422 chunk_header.chunk_sz = rnd_up_len / out->block_size; 423 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 424 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 425 426 if (ret < 0) 427 return -1; 428 ret = out->ops->write(out, data, len); 429 if (ret < 0) 430 return -1; 431 if (zero_len) { 432 ret = out->ops->write(out, out->zero_buf, zero_len); 433 if (ret < 0) 434 return -1; 435 } 436 437 if (out->use_crc) { 438 out->crc32 = sparse_crc32(out->crc32, data, len); 439 if (zero_len) 440 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len); 441 } 442 443 out->cur_out_ptr += rnd_up_len; 444 out->chunk_cnt++; 445 446 return 0; 447 } 448 449 int write_sparse_end_chunk(struct output_file *out) 450 { 451 chunk_header_t chunk_header; 452 int ret; 453 454 if (out->use_crc) { 455 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 456 chunk_header.reserved1 = 0; 457 chunk_header.chunk_sz = 0; 458 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 459 460 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header)); 461 if (ret < 0) { 462 return ret; 463 } 464 out->ops->write(out, &out->crc32, 4); 465 if (ret < 0) { 466 return ret; 467 } 468 469 out->chunk_cnt++; 470 } 471 472 return 0; 473 } 474 475 static struct sparse_file_ops sparse_file_ops = { 476 .write_data_chunk = write_sparse_data_chunk, 477 .write_fill_chunk = write_sparse_fill_chunk, 478 .write_skip_chunk = write_sparse_skip_chunk, 479 .write_end_chunk = write_sparse_end_chunk, 480 }; 481 482 static int write_normal_data_chunk(struct output_file *out, unsigned int len, 483 void *data) 484 { 485 int ret; 486 unsigned int rnd_up_len = ALIGN(len, out->block_size); 487 488 ret = out->ops->write(out, data, len); 489 if (ret < 0) { 490 return ret; 491 } 492 493 if (rnd_up_len > len) { 494 ret = out->ops->skip(out, rnd_up_len - len); 495 } 496 497 return ret; 498 } 499 500 static int write_normal_fill_chunk(struct output_file *out, unsigned int len, 501 uint32_t fill_val) 502 { 503 int ret; 504 unsigned int i; 505 unsigned int write_len; 506 507 /* Initialize fill_buf with the fill_val */ 508 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) { 509 out->fill_buf[i] = fill_val; 510 } 511 512 while (len) { 513 write_len = min(len, out->block_size); 514 ret = out->ops->write(out, out->fill_buf, write_len); 515 if (ret < 0) { 516 return ret; 517 } 518 519 len -= write_len; 520 } 521 522 return 0; 523 } 524 525 static int write_normal_skip_chunk(struct output_file *out, int64_t len) 526 { 527 return out->ops->skip(out, len); 528 } 529 530 int write_normal_end_chunk(struct output_file *out) 531 { 532 return out->ops->pad(out, out->len); 533 } 534 535 static struct sparse_file_ops normal_file_ops = { 536 .write_data_chunk = write_normal_data_chunk, 537 .write_fill_chunk = write_normal_fill_chunk, 538 .write_skip_chunk = write_normal_skip_chunk, 539 .write_end_chunk = write_normal_end_chunk, 540 }; 541 542 void output_file_close(struct output_file *out) 543 { 544 out->sparse_ops->write_end_chunk(out); 545 out->ops->close(out); 546 } 547 548 static int output_file_init(struct output_file *out, int block_size, 549 int64_t len, bool sparse, int chunks, bool crc) 550 { 551 int ret; 552 553 out->len = len; 554 out->block_size = block_size; 555 out->cur_out_ptr = 0ll; 556 out->chunk_cnt = 0; 557 out->crc32 = 0; 558 out->use_crc = crc; 559 560 out->zero_buf = calloc(block_size, 1); 561 if (!out->zero_buf) { 562 error_errno("malloc zero_buf"); 563 return -ENOMEM; 564 } 565 566 out->fill_buf = calloc(block_size, 1); 567 if (!out->fill_buf) { 568 error_errno("malloc fill_buf"); 569 ret = -ENOMEM; 570 goto err_fill_buf; 571 } 572 573 if (sparse) { 574 out->sparse_ops = &sparse_file_ops; 575 } else { 576 out->sparse_ops = &normal_file_ops; 577 } 578 579 if (sparse) { 580 sparse_header_t sparse_header = { 581 .magic = SPARSE_HEADER_MAGIC, 582 .major_version = SPARSE_HEADER_MAJOR_VER, 583 .minor_version = SPARSE_HEADER_MINOR_VER, 584 .file_hdr_sz = SPARSE_HEADER_LEN, 585 .chunk_hdr_sz = CHUNK_HEADER_LEN, 586 .blk_sz = out->block_size, 587 .total_blks = DIV_ROUND_UP(out->len, out->block_size), 588 .total_chunks = chunks, 589 .image_checksum = 0 590 }; 591 592 if (out->use_crc) { 593 sparse_header.total_chunks++; 594 } 595 596 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header)); 597 if (ret < 0) { 598 goto err_write; 599 } 600 } 601 602 return 0; 603 604 err_write: 605 free(out->fill_buf); 606 err_fill_buf: 607 free(out->zero_buf); 608 return ret; 609 } 610 611 static struct output_file *output_file_new_gz(void) 612 { 613 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz)); 614 if (!outgz) { 615 error_errno("malloc struct outgz"); 616 return NULL; 617 } 618 619 outgz->out.ops = &gz_file_ops; 620 621 return &outgz->out; 622 } 623 624 static struct output_file *output_file_new_normal(void) 625 { 626 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal)); 627 if (!outn) { 628 error_errno("malloc struct outn"); 629 return NULL; 630 } 631 632 outn->out.ops = &file_ops; 633 634 return &outn->out; 635 } 636 637 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int), 638 void *priv, unsigned int block_size, int64_t len, 639 int gz __unused, int sparse, int chunks, int crc) 640 { 641 int ret; 642 struct output_file_callback *outc; 643 644 outc = calloc(1, sizeof(struct output_file_callback)); 645 if (!outc) { 646 error_errno("malloc struct outc"); 647 return NULL; 648 } 649 650 outc->out.ops = &callback_file_ops; 651 outc->priv = priv; 652 outc->write = write; 653 654 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc); 655 if (ret < 0) { 656 free(outc); 657 return NULL; 658 } 659 660 return &outc->out; 661 } 662 663 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len, 664 int gz, int sparse, int chunks, int crc) 665 { 666 int ret; 667 struct output_file *out; 668 669 if (gz) { 670 out = output_file_new_gz(); 671 } else { 672 out = output_file_new_normal(); 673 } 674 if (!out) { 675 return NULL; 676 } 677 678 out->ops->open(out, fd); 679 680 ret = output_file_init(out, block_size, len, sparse, chunks, crc); 681 if (ret < 0) { 682 free(out); 683 return NULL; 684 } 685 686 return out; 687 } 688 689 /* Write a contiguous region of data blocks from a memory buffer */ 690 int write_data_chunk(struct output_file *out, unsigned int len, void *data) 691 { 692 return out->sparse_ops->write_data_chunk(out, len, data); 693 } 694 695 /* Write a contiguous region of data blocks with a fill value */ 696 int write_fill_chunk(struct output_file *out, unsigned int len, 697 uint32_t fill_val) 698 { 699 return out->sparse_ops->write_fill_chunk(out, len, fill_val); 700 } 701 702 int write_fd_chunk(struct output_file *out, unsigned int len, 703 int fd, int64_t offset) 704 { 705 int ret; 706 int64_t aligned_offset; 707 int aligned_diff; 708 uint64_t buffer_size; 709 char *ptr; 710 711 aligned_offset = offset & ~(4096 - 1); 712 aligned_diff = offset - aligned_offset; 713 buffer_size = (uint64_t)len + (uint64_t)aligned_diff; 714 715 #ifndef _WIN32 716 if (buffer_size > SIZE_MAX) 717 return -E2BIG; 718 char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, 719 aligned_offset); 720 if (data == MAP_FAILED) { 721 return -errno; 722 } 723 ptr = data + aligned_diff; 724 #else 725 off64_t pos; 726 char *data = malloc(len); 727 if (!data) { 728 return -errno; 729 } 730 pos = lseek64(fd, offset, SEEK_SET); 731 if (pos < 0) { 732 free(data); 733 return -errno; 734 } 735 ret = read_all(fd, data, len); 736 if (ret < 0) { 737 free(data); 738 return ret; 739 } 740 ptr = data; 741 #endif 742 743 ret = out->sparse_ops->write_data_chunk(out, len, ptr); 744 745 #ifndef _WIN32 746 munmap(data, buffer_size); 747 #else 748 free(data); 749 #endif 750 751 return ret; 752 } 753 754 /* Write a contiguous region of data blocks from a file */ 755 int write_file_chunk(struct output_file *out, unsigned int len, 756 const char *file, int64_t offset) 757 { 758 int ret; 759 760 int file_fd = open(file, O_RDONLY | O_BINARY); 761 if (file_fd < 0) { 762 return -errno; 763 } 764 765 ret = write_fd_chunk(out, len, file_fd, offset); 766 767 close(file_fd); 768 769 return ret; 770 } 771 772 int write_skip_chunk(struct output_file *out, int64_t len) 773 { 774 return out->sparse_ops->write_skip_chunk(out, len); 775 } 776
