Home | History | Annotate | Download | only in skqp 
      1 #! /bin/sh
      2 
      3 # Copyright 2018 Google Inc.
      4 # Use of this source code is governed by a BSD-style license that can be
      5 # found in the LICENSE file.
      6 
      7 set -e
      8 
      9 BASE_DIR='platform_tools/android/apps/skqp/src/main/assets'
     10 PATH_LIST='gmkb'
     11 BUCKET=skia-skqp-assets
     12 
     13 EXTANT="$(mktemp "${TMPDIR:-/tmp}/extant.XXXXXXXXXX")"
     14 gsutil ls gs://$BUCKET/ | sed "s|^gs://$BUCKET/||"  > "$EXTANT"
     15 
     16 upload() {
     17     MD5=$(md5sum < "$1" | head -c 32)
     18     if ! grep -q "$MD5" "$EXTANT"; then
     19         URL="gs://${BUCKET}/$MD5"
     20         gsutil cp "$1" "$URL" > /dev/null 2>&1 &
     21     fi
     22     echo $MD5
     23 }
     24 
     25 size() { gsutil du -s gs://$BUCKET | awk '{print $1}'; }
     26 
     27 cd "$(dirname "$0")/../../$BASE_DIR"
     28 
     29 rm -f files.checksum
     30 
     31 FILES="$(mktemp "${TMPDIR:-/tmp}/files.XXXXXXXXXX")"
     32 
     33 : > "$FILES"
     34 
     35 COUNT=$(find $PATH_LIST -type f | wc -l)
     36 INDEX=1
     37 SHARD_COUNT=32
     38 
     39 SIZE=$(size)
     40 find $PATH_LIST -type f | sort | while IFS= read -r FILENAME; do
     41     printf '\r %d / %d   ' "$INDEX" "$COUNT"
     42     if ! [ -f "$FILENAME" ]; then
     43         echo error [${FILENAME}] >&2;
     44         exit 1;
     45     fi
     46     case "$FILENAME" in *\;*) echo bad filename: $FILENAME >&2; exit 1;; esac
     47     MD5=$(upload "$FILENAME")
     48     printf '%s;%s\n' "$MD5" "$FILENAME" >> "$FILES"
     49 
     50     if [ $(($INDEX % $SHARD_COUNT)) = 0 ]; then
     51         wait
     52     fi
     53     INDEX=$(( $INDEX + 1))
     54 done
     55 printf '\rdone          \n'
     56 upload "$FILES" > files.checksum
     57 wait
     58 
     59 D=$(( $(size) - $SIZE ))
     60 printf 'Added %d bytes to %s, %d%%\n' $D $BUCKET $(( $D * 100 / $SIZE ))
     61 
     62 rm "$EXTANT"
     63