1 #!/bin/bash 2 3 # Copyright (c) 2011 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 # This script can be used by waterfall sheriffs to fetch the status 8 # of Valgrind bots on the memory waterfall and test if their local 9 # suppressions match the reports on the waterfall. 10 11 set -e 12 13 THISDIR=$(dirname "${0}") 14 LOGS_DIR=$THISDIR/waterfall.tmp 15 WATERFALL_PAGE="http://build.chromium.org/p/chromium.memory/builders" 16 WATERFALL_FYI_PAGE="http://build.chromium.org/p/chromium.memory.fyi/builders" 17 18 download() { 19 # Download a file. 20 # $1 = URL to download 21 # $2 = Path to the output file 22 # {{{1 23 if [ "$(which curl)" != "" ] 24 then 25 if ! curl -s -o "$2" "$1" 26 then 27 echo 28 echo "Failed to download '$1'... aborting" 29 exit 1 30 fi 31 elif [ "$(which wget)" != "" ] 32 then 33 if ! wget "$1" -O "$2" -q 34 then 35 echo 36 echo "Failed to download '$1'... aborting" 37 exit 1 38 fi 39 else 40 echo "Need either curl or wget to download stuff... aborting" 41 exit 1 42 fi 43 # }}} 44 } 45 46 fetch_logs() { 47 # Fetch Valgrind logs from the waterfall {{{1 48 49 # TODO(timurrrr,maruel): use JSON, see 50 # http://build.chromium.org/p/chromium.memory/json/help 51 52 rm -rf "$LOGS_DIR" # Delete old logs 53 mkdir "$LOGS_DIR" 54 55 echo "Fetching the list of builders..." 56 download $1 "$LOGS_DIR/builders" 57 SLAVES=$(grep "<a href=\"builders\/" "$LOGS_DIR/builders" | \ 58 grep 'td class="box"' | \ 59 sed "s/.*<a href=\"builders\///" | sed "s/\".*//" | \ 60 sort | uniq) 61 62 for S in $SLAVES 63 do 64 SLAVE_URL=$1/$S 65 SLAVE_NAME=$(echo $S | sed -e "s/%20/ /g" -e "s/%28/(/g" -e "s/%29/)/g") 66 echo -n "Fetching builds by slave '${SLAVE_NAME}'" 67 download $SLAVE_URL?numbuilds=${NUMBUILDS} "$LOGS_DIR/slave_${S}" 68 69 # We speed up the 'fetch' step by skipping the builds/tests which succeeded. 70 # TODO(timurrrr): OTOH, we won't be able to check 71 # if some suppression is not used anymore. 72 # 73 # The awk script here joins the lines ending with </td> to make it possible 74 # to find the failed builds. 75 LIST_OF_BUILDS=$(cat "$LOGS_DIR/slave_$S" | \ 76 awk 'BEGIN { buf = "" } 77 { 78 if ($0 ~ /<\/td>/) { buf = (buf $0); } 79 else { 80 if (buf) { print buf; buf="" } 81 print $0 82 } 83 } 84 END {if (buf) print buf}' | \ 85 grep "success\|failure" | \ 86 head -n $NUMBUILDS | \ 87 grep "failure" | \ 88 grep -v "failed compile" | \ 89 sed "s/.*\/builds\///" | sed "s/\".*//") 90 91 for BUILD in $LIST_OF_BUILDS 92 do 93 # We'll fetch a few tiny URLs now, let's use a temp file. 94 TMPFILE=$(mktemp -t memory_waterfall.XXXXXX) 95 download $SLAVE_URL/builds/$BUILD "$TMPFILE" 96 97 REPORT_FILE="$LOGS_DIR/report_${S}_${BUILD}" 98 rm -f $REPORT_FILE 2>/dev/null || true # make sure it doesn't exist 99 100 REPORT_URLS=$(grep -o "[0-9]\+/steps/memory.*/logs/[0-9A-F]\{16\}" \ 101 "$TMPFILE" \ 102 || true) # `true` is to succeed on empty output 103 FAILED_TESTS=$(grep -o "[0-9]\+/steps/memory.*/logs/[A-Za-z0-9_.]\+" \ 104 "$TMPFILE" | grep -v "[0-9A-F]\{16\}" \ 105 | grep -v "stdio" || true) 106 107 for REPORT in $REPORT_URLS 108 do 109 download "$SLAVE_URL/builds/$REPORT/text" "$TMPFILE" 110 echo "" >> "$TMPFILE" # Add a newline at the end 111 cat "$TMPFILE" | tr -d '\r' >> "$REPORT_FILE" 112 done 113 114 for FAILURE in $FAILED_TESTS 115 do 116 echo -n "FAILED:" >> "$REPORT_FILE" 117 echo "$FAILURE" | sed -e "s/.*\/logs\///" -e "s/\/.*//" \ 118 >> "$REPORT_FILE" 119 done 120 121 rm "$TMPFILE" 122 echo $SLAVE_URL/builds/$BUILD >> "$REPORT_FILE" 123 done 124 echo " DONE" 125 done 126 # }}} 127 } 128 129 match_suppressions() { 130 PYTHONPATH=$THISDIR/../python/google \ 131 python "$THISDIR/test_suppressions.py" $@ "$LOGS_DIR/report_"* 132 } 133 134 match_gtest_excludes() { 135 for PLATFORM in "Linux" "Chromium%20Mac" "Chromium%20OS" 136 do 137 echo 138 echo "Test failures on ${PLATFORM}:" | sed "s/%20/ /" 139 grep -h -o "^FAILED:.*" -R "$LOGS_DIR"/*${PLATFORM}* | \ 140 grep -v "FAILS\|FLAKY" | sort | uniq | \ 141 sed -e "s/^FAILED://" -e "s/^/ /" 142 # Don't put any operators between "grep | sed" and "RESULT=$PIPESTATUS" 143 RESULT=$PIPESTATUS 144 145 if [ "$RESULT" == 1 ] 146 then 147 echo " None!" 148 else 149 echo 150 echo " Note: we don't check for failures already excluded locally yet" 151 echo " TODO(timurrrr): don't list tests we've already excluded locally" 152 fi 153 done 154 echo 155 echo "Note: we don't print FAILS/FLAKY tests and 1200s-timeout failures" 156 } 157 158 usage() { 159 cat <<EOF 160 usage: $0 fetch|match options 161 162 This script can be used by waterfall sheriffs to fetch the status 163 of Valgrind bots on the memory waterfall and test if their local 164 suppressions match the reports on the waterfall. 165 166 OPTIONS: 167 -h Show this message 168 -n N Fetch N builds from each slave. 169 170 COMMANDS: 171 fetch Fetch Valgrind logs from the memory waterfall 172 match Test the local suppression files against the downloaded logs 173 174 EOF 175 } 176 177 NUMBUILDS=3 178 179 CMD=$1 180 if [ $# != 0 ]; then 181 shift 182 fi 183 184 # Arguments for "match" are handled in match_suppressions 185 if [ "$CMD" != "match" ]; then 186 while getopts hn: OPTION 187 do 188 case $OPTION in 189 h) 190 usage 191 exit 192 ;; 193 n) 194 NUMBUILDS=$OPTARG 195 ;; 196 ?) 197 usage 198 exit 199 ;; 200 esac 201 done 202 shift $((OPTIND-1)) 203 if [ $# != 0 ]; then 204 usage 205 exit 1 206 fi 207 fi 208 209 if [ "$CMD" = "fetch" ]; then 210 echo "Fetching $NUMBUILDS builds" 211 fetch_logs $WATERFALL_PAGE 212 fetch_logs $WATERFALL_FYI_PAGE 213 elif [ "$CMD" = "match" ]; then 214 match_suppressions $@ 215 match_gtest_excludes 216 elif [ "$CMD" = "blame" ]; then 217 echo The blame command died of bitrot. If you need it, please reimplement it. 218 echo Reimplementation is blocked on http://crbug.com/82688 219 else 220 usage 221 exit 1 222 fi 223
