1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "ext4_utils.h" 18 #include "output_file.h" 19 #include "sparse_format.h" 20 #include "sparse_crc32.h" 21 #include "wipe.h" 22 23 #include <fcntl.h> 24 #include <stdbool.h> 25 #include <sys/types.h> 26 #include <sys/stat.h> 27 #include <sys/types.h> 28 #include <unistd.h> 29 #include <zlib.h> 30 31 #ifndef USE_MINGW 32 #include <sys/mman.h> 33 #define O_BINARY 0 34 #endif 35 36 37 #if defined(__APPLE__) && defined(__MACH__) 38 #define lseek64 lseek 39 #define off64_t off_t 40 #endif 41 42 #define SPARSE_HEADER_MAJOR_VER 1 43 #define SPARSE_HEADER_MINOR_VER 0 44 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t)) 45 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t)) 46 47 struct output_file_ops { 48 int (*seek)(struct output_file *, off64_t); 49 int (*write)(struct output_file *, u8 *, int); 50 void (*close)(struct output_file *); 51 }; 52 53 struct output_file { 54 int fd; 55 gzFile gz_fd; 56 bool close_fd; 57 int sparse; 58 u64 cur_out_ptr; 59 u32 chunk_cnt; 60 u32 crc32; 61 struct output_file_ops *ops; 62 int use_crc; 63 }; 64 65 static int file_seek(struct output_file *out, off64_t off) 66 { 67 off64_t ret; 68 69 ret = lseek64(out->fd, off, SEEK_SET); 70 if (ret < 0) { 71 error_errno("lseek64"); 72 return -1; 73 } 74 return 0; 75 } 76 77 static int file_write(struct output_file *out, u8 *data, int len) 78 { 79 int ret; 80 ret = write(out->fd, data, len); 81 if (ret < 0) { 82 error_errno("write"); 83 return -1; 84 } else if (ret < len) { 85 error("incomplete write"); 86 return -1; 87 } 88 89 return 0; 90 } 91 92 static void file_close(struct output_file *out) 93 { 94 if (out->close_fd) { 95 close(out->fd); 96 } 97 } 98 99 100 static struct output_file_ops file_ops = { 101 .seek = file_seek, 102 .write = file_write, 103 .close = file_close, 104 }; 105 106 static int gz_file_seek(struct output_file *out, off64_t off) 107 { 108 off64_t ret; 109 110 ret = gzseek(out->gz_fd, off, SEEK_SET); 111 if (ret < 0) { 112 error_errno("gzseek"); 113 return -1; 114 } 115 return 0; 116 } 117 118 static int gz_file_write(struct output_file *out, u8 *data, int len) 119 { 120 int ret; 121 ret = gzwrite(out->gz_fd, data, len); 122 if (ret < 0) { 123 error_errno("gzwrite"); 124 return -1; 125 } else if (ret < len) { 126 error("incomplete gzwrite"); 127 return -1; 128 } 129 130 return 0; 131 } 132 133 static void gz_file_close(struct output_file *out) 134 { 135 gzclose(out->gz_fd); 136 } 137 138 static struct output_file_ops gz_file_ops = { 139 .seek = gz_file_seek, 140 .write = gz_file_write, 141 .close = gz_file_close, 142 }; 143 144 static sparse_header_t sparse_header = { 145 .magic = SPARSE_HEADER_MAGIC, 146 .major_version = SPARSE_HEADER_MAJOR_VER, 147 .minor_version = SPARSE_HEADER_MINOR_VER, 148 .file_hdr_sz = SPARSE_HEADER_LEN, 149 .chunk_hdr_sz = CHUNK_HEADER_LEN, 150 .blk_sz = 0, 151 .total_blks = 0, 152 .total_chunks = 0, 153 .image_checksum = 0 154 }; 155 156 static u8 *zero_buf; 157 158 static int emit_skip_chunk(struct output_file *out, u64 skip_len) 159 { 160 chunk_header_t chunk_header; 161 int ret, chunk; 162 163 //DBG printf("skip chunk: 0x%llx bytes\n", skip_len); 164 165 if (skip_len % info.block_size) { 166 error("don't care size %llu is not a multiple of the block size %u", 167 skip_len, info.block_size); 168 return -1; 169 } 170 171 /* We are skipping data, so emit a don't care chunk. */ 172 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE; 173 chunk_header.reserved1 = 0; 174 chunk_header.chunk_sz = skip_len / info.block_size; 175 chunk_header.total_sz = CHUNK_HEADER_LEN; 176 ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 177 if (ret < 0) 178 return -1; 179 180 out->cur_out_ptr += skip_len; 181 out->chunk_cnt++; 182 183 return 0; 184 } 185 186 static int write_chunk_fill(struct output_file *out, u64 off, u32 fill_val, int len) 187 { 188 chunk_header_t chunk_header; 189 int rnd_up_len, zero_len, count; 190 int ret; 191 unsigned int i; 192 u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ 193 194 /* We can assume that all the chunks to be written are in 195 * ascending order, block-size aligned, and non-overlapping. 196 * So, if the offset is less than the current output pointer, 197 * throw an error, and if there is a gap, emit a "don't care" 198 * chunk. The first write (of the super block) may not be 199 * blocksize aligned, so we need to deal with that too. 200 */ 201 //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); 202 203 if (off < out->cur_out_ptr) { 204 error("offset %llu is less than the current output offset %llu", 205 off, out->cur_out_ptr); 206 return -1; 207 } 208 209 if (off > out->cur_out_ptr) { 210 emit_skip_chunk(out, off - out->cur_out_ptr); 211 } 212 213 if (off % info.block_size) { 214 error("write chunk offset %llu is not a multiple of the block size %u", 215 off, info.block_size); 216 return -1; 217 } 218 219 if (off != out->cur_out_ptr) { 220 error("internal error, offset accounting screwy in write_chunk_raw()"); 221 return -1; 222 } 223 224 /* Round up the file length to a multiple of the block size */ 225 rnd_up_len = (len + (info.block_size - 1)) & (~(info.block_size -1)); 226 227 /* Finally we can safely emit a chunk of data */ 228 chunk_header.chunk_type = CHUNK_TYPE_FILL; 229 chunk_header.reserved1 = 0; 230 chunk_header.chunk_sz = rnd_up_len / info.block_size; 231 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val); 232 ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 233 234 if (ret < 0) 235 return -1; 236 ret = out->ops->write(out, (u8 *)&fill_val, sizeof(fill_val)); 237 if (ret < 0) 238 return -1; 239 240 if (out->use_crc) { 241 /* Initialize fill_buf with the fill_val */ 242 for (i = 0; i < (info.block_size / sizeof(u32)); i++) { 243 fill_buf[i] = fill_val; 244 } 245 246 count = chunk_header.chunk_sz; 247 while (count) { 248 out->crc32 = sparse_crc32(out->crc32, fill_buf, info.block_size); 249 count--; 250 } 251 } 252 253 out->cur_out_ptr += rnd_up_len; 254 out->chunk_cnt++; 255 256 return 0; 257 } 258 259 static int write_chunk_raw(struct output_file *out, u64 off, u8 *data, int len) 260 { 261 chunk_header_t chunk_header; 262 int rnd_up_len, zero_len; 263 int ret; 264 265 /* We can assume that all the chunks to be written are in 266 * ascending order, block-size aligned, and non-overlapping. 267 * So, if the offset is less than the current output pointer, 268 * throw an error, and if there is a gap, emit a "don't care" 269 * chunk. The first write (of the super block) may not be 270 * blocksize aligned, so we need to deal with that too. 271 */ 272 //DBG printf("write chunk: offset 0x%llx, length 0x%x bytes\n", off, len); 273 274 if (off < out->cur_out_ptr) { 275 error("offset %llu is less than the current output offset %llu", 276 off, out->cur_out_ptr); 277 return -1; 278 } 279 280 if (off > out->cur_out_ptr) { 281 emit_skip_chunk(out, off - out->cur_out_ptr); 282 } 283 284 if (off % info.block_size) { 285 error("write chunk offset %llu is not a multiple of the block size %u", 286 off, info.block_size); 287 return -1; 288 } 289 290 if (off != out->cur_out_ptr) { 291 error("internal error, offset accounting screwy in write_chunk_raw()"); 292 return -1; 293 } 294 295 /* Round up the file length to a multiple of the block size */ 296 rnd_up_len = (len + (info.block_size - 1)) & (~(info.block_size -1)); 297 zero_len = rnd_up_len - len; 298 299 /* Finally we can safely emit a chunk of data */ 300 chunk_header.chunk_type = CHUNK_TYPE_RAW; 301 chunk_header.reserved1 = 0; 302 chunk_header.chunk_sz = rnd_up_len / info.block_size; 303 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len; 304 ret = out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 305 306 if (ret < 0) 307 return -1; 308 ret = out->ops->write(out, data, len); 309 if (ret < 0) 310 return -1; 311 if (zero_len) { 312 ret = out->ops->write(out, zero_buf, zero_len); 313 if (ret < 0) 314 return -1; 315 } 316 317 if (out->use_crc) { 318 out->crc32 = sparse_crc32(out->crc32, data, len); 319 if (zero_len) 320 out->crc32 = sparse_crc32(out->crc32, zero_buf, zero_len); 321 } 322 323 out->cur_out_ptr += rnd_up_len; 324 out->chunk_cnt++; 325 326 return 0; 327 } 328 329 void close_output_file(struct output_file *out) 330 { 331 int ret; 332 chunk_header_t chunk_header; 333 334 if (out->sparse) { 335 if (out->use_crc) { 336 chunk_header.chunk_type = CHUNK_TYPE_CRC32; 337 chunk_header.reserved1 = 0; 338 chunk_header.chunk_sz = 0; 339 chunk_header.total_sz = CHUNK_HEADER_LEN + 4; 340 341 out->ops->write(out, (u8 *)&chunk_header, sizeof(chunk_header)); 342 out->ops->write(out, (u8 *)&out->crc32, 4); 343 344 out->chunk_cnt++; 345 } 346 347 if (out->chunk_cnt != sparse_header.total_chunks) 348 error("sparse chunk count did not match: %d %d", out->chunk_cnt, 349 sparse_header.total_chunks); 350 } 351 out->ops->close(out); 352 } 353 354 struct output_file *open_output_fd(int fd, int gz, int sparse, 355 int chunks, int crc, int wipe) 356 { 357 int ret; 358 struct output_file *out = malloc(sizeof(struct output_file)); 359 if (!out) { 360 error_errno("malloc struct out"); 361 return NULL; 362 } 363 zero_buf = malloc(info.block_size); 364 if (!zero_buf) { 365 error_errno("malloc zero_buf"); 366 free(out); 367 return NULL; 368 } 369 memset(zero_buf, '\0', info.block_size); 370 371 if (gz) { 372 out->ops = &gz_file_ops; 373 out->gz_fd = gzdopen(fd, "wb9"); 374 if (!out->gz_fd) { 375 error_errno("gzopen"); 376 free(out); 377 return NULL; 378 } 379 } else { 380 out->fd = fd; 381 out->ops = &file_ops; 382 } 383 out->close_fd = false; 384 out->sparse = sparse; 385 out->cur_out_ptr = 0ll; 386 out->chunk_cnt = 0; 387 388 /* Initialize the crc32 value */ 389 out->crc32 = 0; 390 out->use_crc = crc; 391 392 if (wipe) 393 wipe_block_device(out->fd, info.len); 394 395 if (out->sparse) { 396 sparse_header.blk_sz = info.block_size, 397 sparse_header.total_blks = info.len / info.block_size, 398 sparse_header.total_chunks = chunks; 399 if (out->use_crc) 400 sparse_header.total_chunks++; 401 402 ret = out->ops->write(out, (u8 *)&sparse_header, sizeof(sparse_header)); 403 if (ret < 0) 404 return NULL; 405 } 406 407 return out; 408 } 409 410 struct output_file *open_output_file(const char *filename, int gz, int sparse, 411 int chunks, int crc, int wipe) { 412 413 int fd; 414 struct output_file *file; 415 416 if (strcmp(filename, "-")) { 417 fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644); 418 if (fd < 0) { 419 error_errno("open"); 420 return NULL; 421 } 422 } else { 423 fd = STDOUT_FILENO; 424 } 425 426 file = open_output_fd(fd, gz, sparse, chunks, crc, wipe); 427 if (!file) { 428 close(fd); 429 return NULL; 430 } 431 432 file->close_fd = true; // we opened descriptor thus we responsible for closing it 433 434 return file; 435 } 436 437 void pad_output_file(struct output_file *out, u64 len) 438 { 439 int ret; 440 441 if (len > (u64) info.len) { 442 error("attempted to pad file %llu bytes past end of filesystem", 443 len - info.len); 444 return; 445 } 446 if (out->sparse) { 447 /* We need to emit a DONT_CARE chunk to pad out the file if the 448 * cur_out_ptr is not already at the end of the filesystem. 449 */ 450 if (len < out->cur_out_ptr) { 451 error("attempted to pad file %llu bytes less than the current output pointer", 452 out->cur_out_ptr - len); 453 return; 454 } 455 if (len > out->cur_out_ptr) { 456 emit_skip_chunk(out, len - out->cur_out_ptr); 457 } 458 } else { 459 //KEN TODO: Fixme. If the filesystem image needs no padding, 460 // this will overwrite the last byte in the file with 0 461 // The answer is to do accounting like the sparse image 462 // code does and know if there is already data there. 463 ret = out->ops->seek(out, len - 1); 464 if (ret < 0) 465 return; 466 467 ret = out->ops->write(out, (u8*)"", 1); 468 if (ret < 0) 469 return; 470 } 471 } 472 473 /* Write a contiguous region of data blocks from a memory buffer */ 474 void write_data_block(struct output_file *out, u64 off, u8 *data, int len) 475 { 476 int ret; 477 478 if (off + len > (u64) info.len) { 479 error("attempted to write block %llu past end of filesystem", 480 off + len - info.len); 481 return; 482 } 483 484 if (out->sparse) { 485 write_chunk_raw(out, off, data, len); 486 } else { 487 ret = out->ops->seek(out, off); 488 if (ret < 0) 489 return; 490 491 ret = out->ops->write(out, data, len); 492 if (ret < 0) 493 return; 494 } 495 } 496 497 /* Write a contiguous region of data blocks with a fill value */ 498 void write_fill_block(struct output_file *out, u64 off, u32 fill_val, int len) 499 { 500 int ret; 501 unsigned int i; 502 int write_len; 503 u32 fill_buf[4096/sizeof(u32)]; /* Maximum size of a block */ 504 505 if (off + len > (u64) info.len) { 506 error("attempted to write block %llu past end of filesystem", 507 off + len - info.len); 508 return; 509 } 510 511 if (out->sparse) { 512 write_chunk_fill(out, off, fill_val, len); 513 } else { 514 /* Initialize fill_buf with the fill_val */ 515 for (i = 0; i < sizeof(fill_buf)/sizeof(u32); i++) { 516 fill_buf[i] = fill_val; 517 } 518 519 ret = out->ops->seek(out, off); 520 if (ret < 0) 521 return; 522 523 while (len) { 524 write_len = (len > (int)sizeof(fill_buf) ? (int)sizeof(fill_buf) : len); 525 ret = out->ops->write(out, (u8 *)fill_buf, write_len); 526 if (ret < 0) { 527 return; 528 } else { 529 len -= write_len; 530 } 531 } 532 } 533 } 534 535 /* Write a contiguous region of data blocks from a file */ 536 void write_data_file(struct output_file *out, u64 off, const char *file, 537 off64_t offset, int len) 538 { 539 int ret; 540 off64_t aligned_offset; 541 int aligned_diff; 542 int buffer_size; 543 544 if (off + len >= (u64) info.len) { 545 error("attempted to write block %llu past end of filesystem", 546 off + len - info.len); 547 return; 548 } 549 550 int file_fd = open(file, O_RDONLY | O_BINARY); 551 if (file_fd < 0) { 552 error_errno("open"); 553 return; 554 } 555 556 aligned_offset = offset & ~(4096 - 1); 557 aligned_diff = offset - aligned_offset; 558 buffer_size = len + aligned_diff; 559 560 #ifndef USE_MINGW 561 u8 *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, file_fd, 562 aligned_offset); 563 if (data == MAP_FAILED) { 564 error_errno("mmap64"); 565 close(file_fd); 566 return; 567 } 568 #else 569 u8 *data = malloc(buffer_size); 570 if (!data) { 571 error_errno("malloc"); 572 close(file_fd); 573 return; 574 } 575 memset(data, 0, buffer_size); 576 #endif 577 578 if (out->sparse) { 579 write_chunk_raw(out, off, data + aligned_diff, len); 580 } else { 581 ret = out->ops->seek(out, off); 582 if (ret < 0) 583 goto err; 584 585 ret = out->ops->write(out, data + aligned_diff, len); 586 if (ret < 0) 587 goto err; 588 } 589 590 err: 591 #ifndef USE_MINGW 592 munmap(data, buffer_size); 593 #else 594 write(file_fd, data, buffer_size); 595 free(data); 596 #endif 597 close(file_fd); 598 } 599