Home | History | Annotate | Download | only in zip
      1 // Copyright 2015 Google Inc. All rights reserved.
      2 //
      3 // Licensed under the Apache License, Version 2.0 (the "License");
      4 // you may not use this file except in compliance with the License.
      5 // You may obtain a copy of the License at
      6 //
      7 //     http://www.apache.org/licenses/LICENSE-2.0
      8 //
      9 // Unless required by applicable law or agreed to in writing, software
     10 // distributed under the License is distributed on an "AS IS" BASIS,
     11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 // See the License for the specific language governing permissions and
     13 // limitations under the License.
     14 
     15 package zip
     16 
     17 import (
     18 	"bytes"
     19 	"compress/flate"
     20 	"errors"
     21 	"fmt"
     22 	"hash/crc32"
     23 	"io"
     24 	"io/ioutil"
     25 	"os"
     26 	"path/filepath"
     27 	"sort"
     28 	"strings"
     29 	"sync"
     30 	"syscall"
     31 	"time"
     32 	"unicode"
     33 
     34 	"github.com/google/blueprint/pathtools"
     35 
     36 	"android/soong/jar"
     37 	"android/soong/third_party/zip"
     38 )
     39 
     40 // Block size used during parallel compression of a single file.
     41 const parallelBlockSize = 1 * 1024 * 1024 // 1MB
     42 
     43 // Minimum file size to use parallel compression. It requires more
     44 // flate.Writer allocations, since we can't change the dictionary
     45 // during Reset
     46 const minParallelFileSize = parallelBlockSize * 6
     47 
     48 // Size of the ZIP compression window (32KB)
     49 const windowSize = 32 * 1024
     50 
     51 type nopCloser struct {
     52 	io.Writer
     53 }
     54 
     55 func (nopCloser) Close() error {
     56 	return nil
     57 }
     58 
     59 type byteReaderCloser struct {
     60 	*bytes.Reader
     61 	io.Closer
     62 }
     63 
     64 type pathMapping struct {
     65 	dest, src string
     66 	zipMethod uint16
     67 }
     68 
     69 type FileArg struct {
     70 	PathPrefixInZip, SourcePrefixToStrip string
     71 	SourceFiles                          []string
     72 	JunkPaths                            bool
     73 	GlobDir                              string
     74 }
     75 
     76 type FileArgsBuilder struct {
     77 	state FileArg
     78 	err   error
     79 	fs    pathtools.FileSystem
     80 
     81 	fileArgs []FileArg
     82 }
     83 
     84 func NewFileArgsBuilder() *FileArgsBuilder {
     85 	return &FileArgsBuilder{
     86 		fs: pathtools.OsFs,
     87 	}
     88 }
     89 
     90 func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder {
     91 	b.state.JunkPaths = v
     92 	b.state.SourcePrefixToStrip = ""
     93 	return b
     94 }
     95 
     96 func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder {
     97 	b.state.JunkPaths = false
     98 	b.state.SourcePrefixToStrip = prefixToStrip
     99 	return b
    100 }
    101 
    102 func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder {
    103 	b.state.PathPrefixInZip = rootPrefix
    104 	return b
    105 }
    106 
    107 func (b *FileArgsBuilder) File(name string) *FileArgsBuilder {
    108 	if b.err != nil {
    109 		return b
    110 	}
    111 
    112 	arg := b.state
    113 	arg.SourceFiles = []string{name}
    114 	b.fileArgs = append(b.fileArgs, arg)
    115 	return b
    116 }
    117 
    118 func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder {
    119 	if b.err != nil {
    120 		return b
    121 	}
    122 
    123 	arg := b.state
    124 	arg.GlobDir = name
    125 	b.fileArgs = append(b.fileArgs, arg)
    126 	return b
    127 }
    128 
    129 func (b *FileArgsBuilder) List(name string) *FileArgsBuilder {
    130 	if b.err != nil {
    131 		return b
    132 	}
    133 
    134 	f, err := b.fs.Open(name)
    135 	if err != nil {
    136 		b.err = err
    137 		return b
    138 	}
    139 	defer f.Close()
    140 
    141 	list, err := ioutil.ReadAll(f)
    142 	if err != nil {
    143 		b.err = err
    144 		return b
    145 	}
    146 
    147 	arg := b.state
    148 	arg.SourceFiles = strings.Split(string(list), "\n")
    149 	b.fileArgs = append(b.fileArgs, arg)
    150 	return b
    151 }
    152 
    153 func (b *FileArgsBuilder) Error() error {
    154 	if b == nil {
    155 		return nil
    156 	}
    157 	return b.err
    158 }
    159 
    160 func (b *FileArgsBuilder) FileArgs() []FileArg {
    161 	if b == nil {
    162 		return nil
    163 	}
    164 	return b.fileArgs
    165 }
    166 
    167 type IncorrectRelativeRootError struct {
    168 	RelativeRoot string
    169 	Path         string
    170 }
    171 
    172 func (x IncorrectRelativeRootError) Error() string {
    173 	return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot)
    174 }
    175 
    176 type ZipWriter struct {
    177 	time         time.Time
    178 	createdFiles map[string]string
    179 	createdDirs  map[string]string
    180 	directories  bool
    181 
    182 	errors   chan error
    183 	writeOps chan chan *zipEntry
    184 
    185 	cpuRateLimiter    *CPURateLimiter
    186 	memoryRateLimiter *MemoryRateLimiter
    187 
    188 	compressorPool sync.Pool
    189 	compLevel      int
    190 
    191 	followSymlinks     pathtools.ShouldFollowSymlinks
    192 	ignoreMissingFiles bool
    193 
    194 	stderr io.Writer
    195 	fs     pathtools.FileSystem
    196 }
    197 
    198 type zipEntry struct {
    199 	fh *zip.FileHeader
    200 
    201 	// List of delayed io.Reader
    202 	futureReaders chan chan io.Reader
    203 
    204 	// Only used for passing into the MemoryRateLimiter to ensure we
    205 	// release as much memory as much as we request
    206 	allocatedSize int64
    207 }
    208 
    209 type ZipArgs struct {
    210 	FileArgs                 []FileArg
    211 	OutputFilePath           string
    212 	EmulateJar               bool
    213 	AddDirectoryEntriesToZip bool
    214 	CompressionLevel         int
    215 	ManifestSourcePath       string
    216 	NumParallelJobs          int
    217 	NonDeflatedFiles         map[string]bool
    218 	WriteIfChanged           bool
    219 	StoreSymlinks            bool
    220 	IgnoreMissingFiles       bool
    221 
    222 	Stderr     io.Writer
    223 	Filesystem pathtools.FileSystem
    224 }
    225 
    226 const NOQUOTE = '\x00'
    227 
    228 func ReadRespFile(bytes []byte) []string {
    229 	var args []string
    230 	var arg []rune
    231 
    232 	isEscaping := false
    233 	quotingStart := NOQUOTE
    234 	for _, c := range string(bytes) {
    235 		switch {
    236 		case isEscaping:
    237 			if quotingStart == '"' {
    238 				if !(c == '"' || c == '\\') {
    239 					// '\"' or '\\' will be escaped under double quoting.
    240 					arg = append(arg, '\\')
    241 				}
    242 			}
    243 			arg = append(arg, c)
    244 			isEscaping = false
    245 		case c == '\\' && quotingStart != '\'':
    246 			isEscaping = true
    247 		case quotingStart == NOQUOTE && (c == '\'' || c == '"'):
    248 			quotingStart = c
    249 		case quotingStart != NOQUOTE && c == quotingStart:
    250 			quotingStart = NOQUOTE
    251 		case quotingStart == NOQUOTE && unicode.IsSpace(c):
    252 			// Current character is a space outside quotes
    253 			if len(arg) != 0 {
    254 				args = append(args, string(arg))
    255 			}
    256 			arg = arg[:0]
    257 		default:
    258 			arg = append(arg, c)
    259 		}
    260 	}
    261 
    262 	if len(arg) != 0 {
    263 		args = append(args, string(arg))
    264 	}
    265 
    266 	return args
    267 }
    268 
    269 func ZipTo(args ZipArgs, w io.Writer) error {
    270 	if args.EmulateJar {
    271 		args.AddDirectoryEntriesToZip = true
    272 	}
    273 
    274 	// Have Glob follow symlinks if they are not being stored as symlinks in the zip file.
    275 	followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
    276 
    277 	z := &ZipWriter{
    278 		time:               jar.DefaultTime,
    279 		createdDirs:        make(map[string]string),
    280 		createdFiles:       make(map[string]string),
    281 		directories:        args.AddDirectoryEntriesToZip,
    282 		compLevel:          args.CompressionLevel,
    283 		followSymlinks:     followSymlinks,
    284 		ignoreMissingFiles: args.IgnoreMissingFiles,
    285 		stderr:             args.Stderr,
    286 		fs:                 args.Filesystem,
    287 	}
    288 
    289 	if z.fs == nil {
    290 		z.fs = pathtools.OsFs
    291 	}
    292 
    293 	if z.stderr == nil {
    294 		z.stderr = os.Stderr
    295 	}
    296 
    297 	pathMappings := []pathMapping{}
    298 
    299 	noCompression := args.CompressionLevel == 0
    300 
    301 	for _, fa := range args.FileArgs {
    302 		var srcs []string
    303 		for _, s := range fa.SourceFiles {
    304 			s = strings.TrimSpace(s)
    305 			if s == "" {
    306 				continue
    307 			}
    308 
    309 			globbed, _, err := z.fs.Glob(s, nil, followSymlinks)
    310 			if err != nil {
    311 				return err
    312 			}
    313 			if len(globbed) == 0 {
    314 				err := &os.PathError{
    315 					Op:   "lstat",
    316 					Path: s,
    317 					Err:  os.ErrNotExist,
    318 				}
    319 				if args.IgnoreMissingFiles {
    320 					fmt.Fprintln(z.stderr, "warning:", err)
    321 				} else {
    322 					return err
    323 				}
    324 			}
    325 			srcs = append(srcs, globbed...)
    326 		}
    327 		if fa.GlobDir != "" {
    328 			if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
    329 				return err
    330 			} else if !exists && !args.IgnoreMissingFiles {
    331 				err := &os.PathError{
    332 					Op:   "lstat",
    333 					Path: fa.GlobDir,
    334 					Err:  os.ErrNotExist,
    335 				}
    336 				if args.IgnoreMissingFiles {
    337 					fmt.Fprintln(z.stderr, "warning:", err)
    338 				} else {
    339 					return err
    340 				}
    341 			} else if !isDir && !args.IgnoreMissingFiles {
    342 				err := &os.PathError{
    343 					Op:   "lstat",
    344 					Path: fa.GlobDir,
    345 					Err:  syscall.ENOTDIR,
    346 				}
    347 				if args.IgnoreMissingFiles {
    348 					fmt.Fprintln(z.stderr, "warning:", err)
    349 				} else {
    350 					return err
    351 				}
    352 			}
    353 			globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
    354 			if err != nil {
    355 				return err
    356 			}
    357 			srcs = append(srcs, globbed...)
    358 		}
    359 		for _, src := range srcs {
    360 			err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
    361 			if err != nil {
    362 				return err
    363 			}
    364 		}
    365 	}
    366 
    367 	return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs)
    368 }
    369 
    370 func Zip(args ZipArgs) error {
    371 	if args.OutputFilePath == "" {
    372 		return fmt.Errorf("output file path must be nonempty")
    373 	}
    374 
    375 	buf := &bytes.Buffer{}
    376 	var out io.Writer = buf
    377 
    378 	if !args.WriteIfChanged {
    379 		f, err := os.Create(args.OutputFilePath)
    380 		if err != nil {
    381 			return err
    382 		}
    383 
    384 		defer f.Close()
    385 		defer func() {
    386 			if err != nil {
    387 				os.Remove(args.OutputFilePath)
    388 			}
    389 		}()
    390 
    391 		out = f
    392 	}
    393 
    394 	err := ZipTo(args, out)
    395 	if err != nil {
    396 		return err
    397 	}
    398 
    399 	if args.WriteIfChanged {
    400 		err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666)
    401 		if err != nil {
    402 			return err
    403 		}
    404 	}
    405 
    406 	return nil
    407 }
    408 
    409 func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping,
    410 	nonDeflatedFiles map[string]bool, noCompression bool) error {
    411 
    412 	var dest string
    413 
    414 	if fa.JunkPaths {
    415 		dest = filepath.Base(src)
    416 	} else {
    417 		var err error
    418 		dest, err = filepath.Rel(fa.SourcePrefixToStrip, src)
    419 		if err != nil {
    420 			return err
    421 		}
    422 		if strings.HasPrefix(dest, "../") {
    423 			return IncorrectRelativeRootError{
    424 				Path:         src,
    425 				RelativeRoot: fa.SourcePrefixToStrip,
    426 			}
    427 		}
    428 
    429 	}
    430 	dest = filepath.Join(fa.PathPrefixInZip, dest)
    431 
    432 	zipMethod := zip.Deflate
    433 	if _, found := nonDeflatedFiles[dest]; found || noCompression {
    434 		zipMethod = zip.Store
    435 	}
    436 	*pathMappings = append(*pathMappings,
    437 		pathMapping{dest: dest, src: src, zipMethod: zipMethod})
    438 
    439 	return nil
    440 }
    441 
    442 func jarSort(mappings []pathMapping) {
    443 	less := func(i int, j int) (smaller bool) {
    444 		return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
    445 	}
    446 	sort.SliceStable(mappings, less)
    447 }
    448 
    449 func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error {
    450 	z.errors = make(chan error)
    451 	defer close(z.errors)
    452 
    453 	// This channel size can be essentially unlimited -- it's used as a fifo
    454 	// queue decouple the CPU and IO loads. Directories don't require any
    455 	// compression time, but still cost some IO. Similar with small files that
    456 	// can be very fast to compress. Some files that are more difficult to
    457 	// compress won't take a corresponding longer time writing out.
    458 	//
    459 	// The optimum size here depends on your CPU and IO characteristics, and
    460 	// the the layout of your zip file. 1000 was chosen mostly at random as
    461 	// something that worked reasonably well for a test file.
    462 	//
    463 	// The RateLimit object will put the upper bounds on the number of
    464 	// parallel compressions and outstanding buffers.
    465 	z.writeOps = make(chan chan *zipEntry, 1000)
    466 	z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
    467 	z.memoryRateLimiter = NewMemoryRateLimiter(0)
    468 	defer func() {
    469 		z.cpuRateLimiter.Stop()
    470 		z.memoryRateLimiter.Stop()
    471 	}()
    472 
    473 	if manifest != "" && !emulateJar {
    474 		return errors.New("must specify --jar when specifying a manifest via -m")
    475 	}
    476 
    477 	if emulateJar {
    478 		// manifest may be empty, in which case addManifest will fill in a default
    479 		pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
    480 
    481 		jarSort(pathMappings)
    482 	}
    483 
    484 	go func() {
    485 		var err error
    486 		defer close(z.writeOps)
    487 
    488 		for _, ele := range pathMappings {
    489 			if emulateJar && ele.dest == jar.ManifestFile {
    490 				err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
    491 			} else {
    492 				err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar)
    493 			}
    494 			if err != nil {
    495 				z.errors <- err
    496 				return
    497 			}
    498 		}
    499 	}()
    500 
    501 	zipw := zip.NewWriter(f)
    502 
    503 	var currentWriteOpChan chan *zipEntry
    504 	var currentWriter io.WriteCloser
    505 	var currentReaders chan chan io.Reader
    506 	var currentReader chan io.Reader
    507 	var done bool
    508 
    509 	for !done {
    510 		var writeOpsChan chan chan *zipEntry
    511 		var writeOpChan chan *zipEntry
    512 		var readersChan chan chan io.Reader
    513 
    514 		if currentReader != nil {
    515 			// Only read and process errors
    516 		} else if currentReaders != nil {
    517 			readersChan = currentReaders
    518 		} else if currentWriteOpChan != nil {
    519 			writeOpChan = currentWriteOpChan
    520 		} else {
    521 			writeOpsChan = z.writeOps
    522 		}
    523 
    524 		select {
    525 		case writeOp, ok := <-writeOpsChan:
    526 			if !ok {
    527 				done = true
    528 			}
    529 
    530 			currentWriteOpChan = writeOp
    531 
    532 		case op := <-writeOpChan:
    533 			currentWriteOpChan = nil
    534 
    535 			var err error
    536 			if op.fh.Method == zip.Deflate {
    537 				currentWriter, err = zipw.CreateCompressedHeader(op.fh)
    538 			} else {
    539 				var zw io.Writer
    540 
    541 				op.fh.CompressedSize64 = op.fh.UncompressedSize64
    542 
    543 				zw, err = zipw.CreateHeaderAndroid(op.fh)
    544 				currentWriter = nopCloser{zw}
    545 			}
    546 			if err != nil {
    547 				return err
    548 			}
    549 
    550 			currentReaders = op.futureReaders
    551 			if op.futureReaders == nil {
    552 				currentWriter.Close()
    553 				currentWriter = nil
    554 			}
    555 			z.memoryRateLimiter.Finish(op.allocatedSize)
    556 
    557 		case futureReader, ok := <-readersChan:
    558 			if !ok {
    559 				// Done with reading
    560 				currentWriter.Close()
    561 				currentWriter = nil
    562 				currentReaders = nil
    563 			}
    564 
    565 			currentReader = futureReader
    566 
    567 		case reader := <-currentReader:
    568 			_, err := io.Copy(currentWriter, reader)
    569 			if err != nil {
    570 				return err
    571 			}
    572 
    573 			currentReader = nil
    574 
    575 		case err := <-z.errors:
    576 			return err
    577 		}
    578 	}
    579 
    580 	// One last chance to catch an error
    581 	select {
    582 	case err := <-z.errors:
    583 		return err
    584 	default:
    585 		zipw.Close()
    586 		return nil
    587 	}
    588 }
    589 
    590 // imports (possibly with compression) <src> into the zip at sub-path <dest>
    591 func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error {
    592 	var fileSize int64
    593 	var executable bool
    594 
    595 	var s os.FileInfo
    596 	var err error
    597 	if z.followSymlinks {
    598 		s, err = z.fs.Stat(src)
    599 	} else {
    600 		s, err = z.fs.Lstat(src)
    601 	}
    602 
    603 	if err != nil {
    604 		if os.IsNotExist(err) && z.ignoreMissingFiles {
    605 			fmt.Fprintln(z.stderr, "warning:", err)
    606 			return nil
    607 		}
    608 		return err
    609 	} else if s.IsDir() {
    610 		if z.directories {
    611 			return z.writeDirectory(dest, src, emulateJar)
    612 		}
    613 		return nil
    614 	} else {
    615 		if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
    616 			return err
    617 		}
    618 
    619 		if prev, exists := z.createdDirs[dest]; exists {
    620 			return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
    621 		}
    622 		if prev, exists := z.createdFiles[dest]; exists {
    623 			return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
    624 		}
    625 
    626 		z.createdFiles[dest] = src
    627 
    628 		if s.Mode()&os.ModeSymlink != 0 {
    629 			return z.writeSymlink(dest, src)
    630 		} else if !s.Mode().IsRegular() {
    631 			return fmt.Errorf("%s is not a file, directory, or symlink", src)
    632 		}
    633 
    634 		fileSize = s.Size()
    635 		executable = s.Mode()&0100 != 0
    636 	}
    637 
    638 	r, err := z.fs.Open(src)
    639 	if err != nil {
    640 		return err
    641 	}
    642 
    643 	header := &zip.FileHeader{
    644 		Name:               dest,
    645 		Method:             method,
    646 		UncompressedSize64: uint64(fileSize),
    647 	}
    648 
    649 	if executable {
    650 		header.SetMode(0700)
    651 	}
    652 
    653 	return z.writeFileContents(header, r)
    654 }
    655 
    656 func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
    657 	if prev, exists := z.createdDirs[dest]; exists {
    658 		return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
    659 	}
    660 	if prev, exists := z.createdFiles[dest]; exists {
    661 		return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
    662 	}
    663 
    664 	if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
    665 		return err
    666 	}
    667 
    668 	var contents []byte
    669 	if src != "" {
    670 		f, err := z.fs.Open(src)
    671 		if err != nil {
    672 			return err
    673 		}
    674 
    675 		contents, err = ioutil.ReadAll(f)
    676 		f.Close()
    677 		if err != nil {
    678 			return err
    679 		}
    680 	}
    681 
    682 	fh, buf, err := jar.ManifestFileContents(contents)
    683 	if err != nil {
    684 		return err
    685 	}
    686 
    687 	reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
    688 
    689 	return z.writeFileContents(fh, reader)
    690 }
    691 
    692 func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
    693 
    694 	header.SetModTime(z.time)
    695 
    696 	compressChan := make(chan *zipEntry, 1)
    697 	z.writeOps <- compressChan
    698 
    699 	// Pre-fill a zipEntry, it will be sent in the compressChan once
    700 	// we're sure about the Method and CRC.
    701 	ze := &zipEntry{
    702 		fh: header,
    703 	}
    704 
    705 	ze.allocatedSize = int64(header.UncompressedSize64)
    706 	z.cpuRateLimiter.Request()
    707 	z.memoryRateLimiter.Request(ze.allocatedSize)
    708 
    709 	fileSize := int64(header.UncompressedSize64)
    710 	if fileSize == 0 {
    711 		fileSize = int64(header.UncompressedSize)
    712 	}
    713 
    714 	if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
    715 		wg := new(sync.WaitGroup)
    716 
    717 		// Allocate enough buffer to hold all readers. We'll limit
    718 		// this based on actual buffer sizes in RateLimit.
    719 		ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
    720 
    721 		// Calculate the CRC in the background, since reading the entire
    722 		// file could take a while.
    723 		//
    724 		// We could split this up into chunks as well, but it's faster
    725 		// than the compression. Due to the Go Zip API, we also need to
    726 		// know the result before we can begin writing the compressed
    727 		// data out to the zipfile.
    728 		wg.Add(1)
    729 		go z.crcFile(r, ze, compressChan, wg)
    730 
    731 		for start := int64(0); start < fileSize; start += parallelBlockSize {
    732 			sr := io.NewSectionReader(r, start, parallelBlockSize)
    733 			resultChan := make(chan io.Reader, 1)
    734 			ze.futureReaders <- resultChan
    735 
    736 			z.cpuRateLimiter.Request()
    737 
    738 			last := !(start+parallelBlockSize < fileSize)
    739 			var dict []byte
    740 			if start >= windowSize {
    741 				dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
    742 				if err != nil {
    743 					return err
    744 				}
    745 			}
    746 
    747 			wg.Add(1)
    748 			go z.compressPartialFile(sr, dict, last, resultChan, wg)
    749 		}
    750 
    751 		close(ze.futureReaders)
    752 
    753 		// Close the file handle after all readers are done
    754 		go func(wg *sync.WaitGroup, closer io.Closer) {
    755 			wg.Wait()
    756 			closer.Close()
    757 		}(wg, r)
    758 	} else {
    759 		go func() {
    760 			z.compressWholeFile(ze, r, compressChan)
    761 			r.Close()
    762 		}()
    763 	}
    764 
    765 	return nil
    766 }
    767 
    768 func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
    769 	defer wg.Done()
    770 	defer z.cpuRateLimiter.Finish()
    771 
    772 	crc := crc32.NewIEEE()
    773 	_, err := io.Copy(crc, r)
    774 	if err != nil {
    775 		z.errors <- err
    776 		return
    777 	}
    778 
    779 	ze.fh.CRC32 = crc.Sum32()
    780 	resultChan <- ze
    781 	close(resultChan)
    782 }
    783 
    784 func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
    785 	defer wg.Done()
    786 
    787 	result, err := z.compressBlock(r, dict, last)
    788 	if err != nil {
    789 		z.errors <- err
    790 		return
    791 	}
    792 
    793 	z.cpuRateLimiter.Finish()
    794 
    795 	resultChan <- result
    796 }
    797 
    798 func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
    799 	buf := new(bytes.Buffer)
    800 	var fw *flate.Writer
    801 	var err error
    802 	if len(dict) > 0 {
    803 		// There's no way to Reset a Writer with a new dictionary, so
    804 		// don't use the Pool
    805 		fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
    806 	} else {
    807 		var ok bool
    808 		if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
    809 			fw.Reset(buf)
    810 		} else {
    811 			fw, err = flate.NewWriter(buf, z.compLevel)
    812 		}
    813 		defer z.compressorPool.Put(fw)
    814 	}
    815 	if err != nil {
    816 		return nil, err
    817 	}
    818 
    819 	_, err = io.Copy(fw, r)
    820 	if err != nil {
    821 		return nil, err
    822 	}
    823 	if last {
    824 		fw.Close()
    825 	} else {
    826 		fw.Flush()
    827 	}
    828 
    829 	return buf, nil
    830 }
    831 
    832 func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
    833 
    834 	crc := crc32.NewIEEE()
    835 	_, err := io.Copy(crc, r)
    836 	if err != nil {
    837 		z.errors <- err
    838 		return
    839 	}
    840 
    841 	ze.fh.CRC32 = crc.Sum32()
    842 
    843 	_, err = r.Seek(0, 0)
    844 	if err != nil {
    845 		z.errors <- err
    846 		return
    847 	}
    848 
    849 	readFile := func(reader io.ReadSeeker) ([]byte, error) {
    850 		_, err := reader.Seek(0, 0)
    851 		if err != nil {
    852 			return nil, err
    853 		}
    854 
    855 		buf, err := ioutil.ReadAll(reader)
    856 		if err != nil {
    857 			return nil, err
    858 		}
    859 
    860 		return buf, nil
    861 	}
    862 
    863 	ze.futureReaders = make(chan chan io.Reader, 1)
    864 	futureReader := make(chan io.Reader, 1)
    865 	ze.futureReaders <- futureReader
    866 	close(ze.futureReaders)
    867 
    868 	if ze.fh.Method == zip.Deflate {
    869 		compressed, err := z.compressBlock(r, nil, true)
    870 		if err != nil {
    871 			z.errors <- err
    872 			return
    873 		}
    874 		if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
    875 			futureReader <- compressed
    876 		} else {
    877 			buf, err := readFile(r)
    878 			if err != nil {
    879 				z.errors <- err
    880 				return
    881 			}
    882 			ze.fh.Method = zip.Store
    883 			futureReader <- bytes.NewReader(buf)
    884 		}
    885 	} else {
    886 		buf, err := readFile(r)
    887 		if err != nil {
    888 			z.errors <- err
    889 			return
    890 		}
    891 		ze.fh.Method = zip.Store
    892 		futureReader <- bytes.NewReader(buf)
    893 	}
    894 
    895 	z.cpuRateLimiter.Finish()
    896 
    897 	close(futureReader)
    898 
    899 	compressChan <- ze
    900 	close(compressChan)
    901 }
    902 
    903 // writeDirectory annotates that dir is a directory created for the src file or directory, and adds
    904 // the directory entry to the zip file if directories are enabled.
    905 func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
    906 	// clean the input
    907 	dir = filepath.Clean(dir)
    908 
    909 	// discover any uncreated directories in the path
    910 	zipDirs := []string{}
    911 	for dir != "" && dir != "." {
    912 		if _, exists := z.createdDirs[dir]; exists {
    913 			break
    914 		}
    915 
    916 		if prev, exists := z.createdFiles[dir]; exists {
    917 			return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
    918 		}
    919 
    920 		z.createdDirs[dir] = src
    921 		// parent directories precede their children
    922 		zipDirs = append([]string{dir}, zipDirs...)
    923 
    924 		dir = filepath.Dir(dir)
    925 	}
    926 
    927 	if z.directories {
    928 		// make a directory entry for each uncreated directory
    929 		for _, cleanDir := range zipDirs {
    930 			var dirHeader *zip.FileHeader
    931 
    932 			if emulateJar && cleanDir+"/" == jar.MetaDir {
    933 				dirHeader = jar.MetaDirFileHeader()
    934 			} else {
    935 				dirHeader = &zip.FileHeader{
    936 					Name: cleanDir + "/",
    937 				}
    938 				dirHeader.SetMode(0700 | os.ModeDir)
    939 			}
    940 
    941 			dirHeader.SetModTime(z.time)
    942 
    943 			ze := make(chan *zipEntry, 1)
    944 			ze <- &zipEntry{
    945 				fh: dirHeader,
    946 			}
    947 			close(ze)
    948 			z.writeOps <- ze
    949 		}
    950 	}
    951 
    952 	return nil
    953 }
    954 
    955 func (z *ZipWriter) writeSymlink(rel, file string) error {
    956 	fileHeader := &zip.FileHeader{
    957 		Name: rel,
    958 	}
    959 	fileHeader.SetModTime(z.time)
    960 	fileHeader.SetMode(0777 | os.ModeSymlink)
    961 
    962 	dest, err := z.fs.Readlink(file)
    963 	if err != nil {
    964 		return err
    965 	}
    966 
    967 	fileHeader.UncompressedSize64 = uint64(len(dest))
    968 	fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest))
    969 
    970 	ze := make(chan *zipEntry, 1)
    971 	futureReaders := make(chan chan io.Reader, 1)
    972 	futureReader := make(chan io.Reader, 1)
    973 	futureReaders <- futureReader
    974 	close(futureReaders)
    975 	futureReader <- bytes.NewBufferString(dest)
    976 	close(futureReader)
    977 
    978 	ze <- &zipEntry{
    979 		fh:            fileHeader,
    980 		futureReaders: futureReaders,
    981 	}
    982 	close(ze)
    983 	z.writeOps <- ze
    984 
    985 	return nil
    986 }
    987