1 #!/bin/sh -xe 2 3 SetupCrontab () 4 { 5 echo "Setup crontab." 6 7 set +e 8 crontab -r 9 set -e 10 11 # crontab in some distros will not read from STDIN. 12 13 cat <<EOF >kdump.cron 14 SHELL=/bin/sh 15 PATH=/usr/bin:/usr/sbin:/sbin:/bin 16 MAILTO=root 17 @reboot cd "$(pwd)"; cd ..; ${0} >>/tmp/kdump-$(date +%F-%T).log 2>&1 18 EOF 19 20 crontab kdump.cron 21 22 echo "Enable cron daemon by default." 23 24 if [ -f /etc/init.d/crond ]; then 25 cron=crond 26 else 27 # SUSE 28 cron=cron 29 fi 30 31 # Red Hat and SUSE. 32 if [ -x "/sbin/chkconfig" ]; then 33 /sbin/chkconfig "${cron}" on 34 35 # Debian and Ubuntu. 36 elif [ -x "/sbin/update-rc.d" ]; then 37 /sbin/update-rc.d "${cron}" defaults 38 fi 39 } 40 41 SetupKdump () 42 { 43 echo "Start kdump daemon." 44 /etc/init.d/kdump restart 45 46 echo "Enable kdump daemon by default." 47 # Red Hat and SUSE. 48 if [ -x "/sbin/chkconfig" ]; then 49 /sbin/chkconfig kdump on 50 51 # Debian and Ubuntu. 52 elif [ -x "/sbin/update-rc.d" ]; then 53 /sbin/update-rc.d kdump defaults 54 fi 55 } 56 57 58 PrepareVerify () 59 { 60 if [ "${last}" = "KLEXT" ]; then 61 # If not mountable, skip it, and continue doing the test. 62 set +e 63 mount "${EXT3_PART}" /mnt 64 set -e 65 66 COREDIR=/mnt"${COREDIR}" 67 68 elif [ "${last}" = "KLLBL" ]; then 69 # If not mountable, skip it, and continue doing the test. 70 set +e 71 mount -L "${EXT3_LABEL}" /mnt 72 set -e 73 74 COREDIR=/mnt"${COREDIR}" 75 76 elif [ "${last}" = "KLUID" ]; then 77 # If not mountable, skip it, and continue doing the test. 78 set +e 79 mount "/dev/disk/by-uuid/${EXT3_UID}" /mnt 80 set -e 81 82 COREDIR=/mnt"${COREDIR}" 83 84 elif [ "${last}" = "KLRAW" ]; then 85 mkdir -p "${COREDIR}/${last}" 86 87 # If not dumpable, skip it, and continue doing the test. 88 set +e 89 dd if="${RAW_PART}" of="${COREDIR}/${last}/vmcore" bs=1024 90 set -e 91 92 elif [ "${last}" = "KNSCP" ]; then 93 if [ -z "${SCP_PATH}" ]; then 94 echo "Fail: network destination not defined." 95 exit 1 96 fi 97 98 file=$(ssh -i ~/.ssh/kdump_id_rsa "${SCP_PATH}" "ls -t ${COREDIR}/*/vmcore* \ 99 2>/dev/null | head -1") 100 101 mkdir -p "${COREDIR}/${last}" 102 103 if [ "${file}" ]; then 104 # Not fatal error. 105 set +e 106 scp -i ~/.ssh/kdump_id_rsa "${SCP_PATH}:${file}" "${COREDIR}/${last}" 107 set -e 108 fi 109 110 elif [ "${last}" = "KNNFS" ]; then 111 # Not fatal error. 112 set +e 113 mount "${NFS_PATH}" /mnt 114 set -e 115 116 COREDIR=/mnt"${COREDIR}" 117 fi 118 119 vmcore=$(ls -t "${COREDIR}"/*/vmcore* 2>/dev/null | head -1) 120 } 121 122 VerifyTest () 123 { 124 # Should not be here. 125 if [ -z "${last}" ]; then 126 echo "Should not be here!" 127 echo "There must be something wrong with the test setup." 128 exit 1 129 fi 130 131 echo "Verifying the result of previous test ${last}." 132 ldir=$(ls -td "../${log}/$(hostname)."* | head -1) 133 134 if [ -f "${vmcore}" ]; then 135 echo "$(date +%F-%T): verification of test ${last} passed." \ 136 >>"${ldir}/status" 137 138 ./verify.sh "../${conf}" "${vmcore}" "${CRASH}" \ 139 >>"${ldir}/${ITERATION}.${last}.$(date +%F-%T)" 140 141 # Be careful to define COREDIR. 142 rm -rf "${COREDIR}"/* 143 144 else 145 echo "$(date +%F-%T): verification of test ${last} failed:\ 146 vmcore NOT FOUND." >>"${ldir}/status" 147 echo "vmcore NOT FOUND." \ 148 >>"${ldir}/${ITERATION}.${last}.$(date +%F-%T)" 149 fi 150 } 151 152 RunTest () 153 { 154 155 sed -i "s/\(^REBOOT\)=.*/\1=$((count + 1))/" \ 156 "../${conf}" 157 158 echo "Running current test ${i}." 159 160 echo "$(date +%F-%T): running current test ${i}." \ 161 >> "${ldir}/status" 162 163 # Save STDIO buffers. 164 sync 165 ./test.sh "../${conf}" "${i}" "../${log}" 166 } 167 168 # Start test. 169 conf="./runkdump.conf" 170 lib="lib" 171 log="log" 172 173 # Read test configuration file. 174 . "${conf}" 175 176 # Check mandatory variables. 177 if [ -z "${ITERATION}" ] || [ -z "${REBOOT}" ] || [ -z "${COREDIR}" ] 178 then 179 echo "Fail: some mandatory variables are missing from\ 180 configuration file." 181 exit 1 182 fi 183 184 cd "${lib}" 185 186 while [ "${ITERATION}" -ge 1 ]; do 187 188 # Reboot the machine first to take advantage of boot parameter 189 # changes. 190 if [ -z "${REBOOT}" ] || [ "${REBOOT}" -eq 0 ]; then 191 echo "Setup test environment." 192 193 SetupCrontab 194 195 ./setup.sh "../${conf}" 196 197 sed -i 's/\(^REBOOT\)=.*/\1=1/' "../${conf}" 198 199 echo "System is going to reboot." 200 /sbin/shutdown -r now 201 sleep 60 202 203 else 204 count=1 205 206 for i in ${CRASHER} ${BASIC_LKDTM} ${EXTRA_LKDTM} ${EXTRA_DUMP} \ 207 END; do 208 209 if [ "${count}" -eq "${REBOOT}" ]; then 210 # Wait for machine fully booted. 211 sleep 60 212 213 # First Test. 214 if [ "${REBOOT}" -eq 1 ]; then 215 echo "First test..." 216 echo "Verify Boot Loader." 217 if ! grep 'crashkernel=' /proc/cmdline; then 218 echo "Fail: error changing Boot Loader, no crashkernel=." 219 exit 1 220 fi 221 222 SetupKdump 223 224 # Creat log directory. 225 mkdir -p "../${log}/$(hostname).$(date +%F-%T)" 226 227 echo "Gather system information." 228 229 ldir=$(ls -td "../${log}/$(hostname)."* | head -1) 230 ./sysinfo.sh >"${ldir}/system.info" 231 232 else 233 PrepareVerify 234 235 VerifyTest 236 237 if [ "${i}" = END ]; then 238 # We are done. 239 break 240 fi 241 242 fi 243 244 RunTest 245 246 # Some tests could not reboot target. They can hung up 247 # machine or leave it working. But we need to do all 248 # tests. So we are going to reboot if we are in wrong 249 # place. 250 251 sleep 3600 252 echo "$(date +%F-%T): manually reboot for test ${i}." >>"${ldir}/status" 253 /sbin/shutdown -r now 254 sleep 60 255 fi 256 257 # No test is scheduled to run. 258 count=$((count + 1)) 259 last=${i} 260 done 261 fi 262 263 if [ "${ITERATION}" -eq 1 ]; then 264 # We are done. 265 break 266 267 else 268 # Run the next iteration. 269 sed -i "s/\(^ITERATION\)=.*/\1=$((ITERATION - 1))/" \ 270 "../${conf}" 271 fi 272 273 done 274 275 # We are done. 276 # Reset. 277 sed -i "s/\(^REBOOT\)=.*/\1=0/" "../${conf}" 278 crontab -r 279 ldir=$(ls -td "../${log}/$(hostname)."* | head -1) 280 echo "$(date +%F-%T): test run complete." >>"${ldir}/status" 281 282 exit 0 283
