Home | History | Annotate | Download | only in freezer
      1 #!/bin/bash
      2 # Copyright (c) International Business Machines  Corp., 2008
      3 # Author: Matt Helsley <matthltc (at] us.ibm.com>
      4 #
      5 # This library is free software; you can redistribute it and/or
      6 # modify it under the terms of the GNU Lesser General Public
      7 # License as published by the Free Software Foundation; either
      8 # version 2.1 of the License, or (at your option) any later version.
      9 #
     10 # This library is distributed in the hope that it will be useful,
     11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
     12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     13 # Lesser General Public License for more details.
     14 #
     15 # You should have received a copy of the GNU Lesser General Public
     16 # License along with this library; if not, write to the Free Software
     17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
     18 #
     19 #
     20 
     21 # A library of cgroup test functions for testing the cgroup freezer and,
     22 # if present, a cgroup signal subsystem.
     23 #
     24 # Most of these assume the current directory is the cgroup of interest.
     25 # mount_{freezer|signal} and make_sample_cgroup are the exceptions to this rule.
     26 #
     27 # On failure, a message indicating what failed is printed and the
     28 #    exit status of the failing command is returned. If "result" is unset
     29 #    then we assign the exit status of the failed command to it.
     30 #
     31 # The variable "result" holds the exit status of the first command that failed,
     32 #    $UNFINISHED if no command has failed yet, or $FINISHED if no command
     33 #    has failed and we've finished all significant parts of the test. Note:
     34 #    significant parts usually do not include the cleanup of the test.
     35 #
     36 
     37 
     38 # xargs 4.1.20 only accepts -i instead of -I
     39 # However -I is added and -i deprecated somewhere between (4.1.20, 4.2.32]
     40 XRGSV=$(xargs --version | sed -e 's/^[^[:digit:]]*//')
     41 export XARGS_REPL_STR="%"
     42 case ${XRGSV} in
     43 [456789].[23456789][0-9]*.*|[1-9][0-9][0-9]*.*.*) # version > 4.1.*
     44 	export XARGS_REPL_OPT="-I${XARGS_REPL_STR}"
     45 	;;
     46 4.1.*|*)
     47 	export XARGS_REPL_OPT="-i${XARGS_REPL_STR}"
     48 	;;
     49 esac
     50 unset XRGSV
     51 
     52 export UNFINISHED=""
     53 export FINISHED=0
     54 export TMP=${TMP:-/tmp}
     55 
     56 #
     57 # Tests are either running or cleaning up. Cleanup phases do not emit TFAIL.
     58 #
     59 export LIB_TEST_STATE="running"
     60 
     61 export max_state_samples=5 # number of times to sample
     62 export sample_state_period=1  # number of seconds between samplings
     63 export sample_sleep=5     # how long the sample program should sleep
     64 export result="$UNFINISHED"
     65 
     66 # These are echo'd to freezer.state.
     67 export FREEZE='FROZEN'
     68 export THAW='THAWED'
     69 
     70 # We use /bin/echo to write to cgroup files because it's exit status will not
     71 # hide write errors (bash's echo does not indicate if the write succeeded).
     72 export CG_FILE_WRITE=/bin/echo
     73 
     74 declare -r UNFINISHED FINISHED FREEZE THAW max_state_samples sample_state_period
     75 
     76 # When we're running we want to issue failure results when things go wrong.
     77 function running_cgroup_test()
     78 {
     79 	export LIB_TEST_STATE="TFAIL"
     80 }
     81 
     82 # But when we're cleaning up we want to issue warnings (if not TINFO).
     83 function cleanup_cgroup_test()
     84 {
     85 	export LIB_TEST_STATE="TWARN"
     86 }
     87 
     88 # Mounts the cgroup filesystem somewhere and pushes pwd onto the dir stack
     89 function mount_cgroup_subsys()
     90 {
     91 	local rc=0
     92 
     93 	mkdir -p $TMP/${cgroup_subsys}_test > /dev/null 2>&1
     94 	rc=$?
     95 
     96 	# Don't test status because we don't care if the directory already
     97 	# exists.
     98 
     99 	if [ ! -d $TMP/${cgroup_subsys}_test ]; then
    100 		result=${result:-$rc}
    101 		tst_brkm TBROK "Failed to make mount point for cgroup filesystem"
    102 		return $result
    103 	fi
    104 
    105 	mount -t cgroup -o${cgroup_subsys} test_cgroup_${cgroup_subsys} $TMP/${cgroup_subsys}_test
    106 	rc=$?
    107 	if [ $rc -ne 0 ]; then
    108 		result=${result:-$rc}
    109 		tst_resm TINFO "Failed to mount cgroup filesystem with ${cgroup_subsys} subsystem."
    110 		rmdir $TMP/${cgroup_subsys}_test 2> /dev/null
    111 		return $rc
    112 	fi
    113 
    114 	export mount_cgroup_freezer_saved_dir="$(pwd)"
    115 	cd $TMP/${cgroup_subsys}_test > /dev/null 2>&1
    116 	rc=$?
    117 	if [ $rc -ne 0 ]; then
    118 		result=${result:-$rc}
    119 		tst_brkm TBROK "Failed to change working directory into cgroup filesystem (pwd: \"$(pwd)\")"
    120 		umount $TMP/${cgroup_subsys}_test || umount -l $TMP/${cgroup_subsys}_test || tst_brkm TBROK "Failed to unmount cgroup filesystem with ${cgroup_subsys} subsystem"
    121 		rmdir $TMP/${cgroup_subsys}_test 2> /dev/null
    122 		return $rc
    123 	fi
    124 
    125 	return 0
    126 }
    127 
    128 function mount_freezer()
    129 {
    130 	export cgroup_subsys=freezer
    131 	mount_cgroup_subsys
    132 }
    133 
    134 function mount_signal()
    135 {
    136 	export cgroup_subsys=signal
    137 	mount_cgroup_subsys
    138 }
    139 
    140 # umounts the cgroup filesystem and pops the dir stack
    141 function umount_cgroup_subsys()
    142 {
    143 	cd "${mount_cgroup_freezer_saved_dir}"
    144 	local cwd_result=$?
    145 	umount $TMP/${cgroup_subsys}_test > /dev/null 2>&1 \
    146 	  || umount -l $TMP/${cgroup_subsys}_test || \
    147 	    tst_brkm TBROK "Failed to unmount cgroup filesystem (umount exit status: $?)"
    148 	local umnt_result=$?
    149 	local rc=0
    150 
    151 	if [ $cwd_result -ne 0 ]; then
    152 		result=${result:-$cwd_result}
    153 		rc=$cwd_result
    154 	elif [ $umnt_result -ne 0 ]; then
    155 		result=${result:-$umnt_result}
    156 		rc=$umnt_result
    157 	elif [ $umnt_result -eq 0 -a $cwd_result -eq 0 ]; then
    158 		rmdir $TMP/${cgroup_subsys}_test
    159 		return 0
    160 	fi
    161 
    162 	return $rc
    163 }
    164 
    165 function umount_freezer()
    166 {
    167 	[ "${cgroup_subsys}" != "freezer" ] && {
    168 		result=${result:-5}
    169 		exit -1
    170 	}
    171 	umount_cgroup_subsys
    172 	unset cgroup_subsys
    173 }
    174 
    175 function cleanup_freezer()
    176 {
    177 	local save_result="${result}"
    178 	local save_pwd="$(pwd)"
    179 
    180 	mount_freezer && {
    181 		# Run these commands in bash because we need $(cmd subst) and
    182 		# we need to redirect to different freezer.state files for each
    183 		# group
    184 		# Kill any leftover tasks
    185 		disown -a
    186 		find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | \
    187 		xargs -0r -n 1 ${XARGS_REPL_OPT} /bin/bash -c 'kill $(cat "'"${XARGS_REPL_STR}"'/tasks") 2> /dev/null'
    188 
    189 		# For each group in the freezer hierarch, that its tasks
    190 		find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | \
    191 		xargs -0r -n 1 ${XARGS_REPL_OPT} /bin/bash -c "\"${CG_FILE_WRITE}\" \"${THAW}\" > '${XARGS_REPL_STR}/freezer.state'"
    192 
    193 		# Kill any leftover tasks
    194 		find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | \
    195 		xargs -0r -n 1 ${XARGS_REPL_OPT} /bin/bash -c 'kill $(cat "'"${XARGS_REPL_STR}"'/tasks") 2> /dev/null'
    196 
    197 		sleep 2
    198 
    199 		# Really kill any leftover tasks
    200 		find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | \
    201 		xargs -0r -n 1 ${XARGS_REPL_OPT} /bin/bash -c 'kill -s SIGKILL $(cat "'"${XARGS_REPL_STR}"'/tasks") 2> /dev/null'
    202 
    203 		# Don't need to run these xargs commands in bash since we want
    204 		# to see what's left on stdout
    205 		LINES=$(find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | \
    206 			xargs -0r -n 1 ${XARGS_REPL_OPT} cat "${XARGS_REPL_STR}/tasks" | wc -l)
    207 		if (( LINES == 0 )); then
    208 			# Remove the empty groups
    209 			find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | xargs -r0 rmdir
    210 		else
    211 			tst_resm TWARN "Could not cleanup:"
    212 			find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | xargs -0r -n 1 ${XARGS_REPL_OPT} ls -ld "${XARGS_REPL_STR}/tasks"
    213 		fi
    214 
    215 		umount_freezer
    216 	}
    217 
    218 	if [ "$save_pwd" != `pwd` ]; then
    219 		tst_resm TWARN "libcgroup_subsys: cleanup_freezer() is broken"
    220 		cd "$save_pwd"
    221 	fi
    222 
    223 	result="${save_result}"
    224 }
    225 
    226 function umount_signal()
    227 {
    228 	[ "${cgroup_subsys}" != "signal" ] && {
    229 		result=${result:-6}
    230 		exit -1
    231 	}
    232 	umount_cgroup_subsys
    233 	unset cgroup_subsys
    234 }
    235 
    236 function cleanup_signal()
    237 {
    238 	local save_result="${result}"
    239 	local save_pwd="$(pwd)"
    240 
    241 	mount_signal && {
    242 		find $TMP/${cgroup_subsys}_test -mindepth 1 -depth -type d -print0 | xargs -r0 rmdir
    243 		umount_signal
    244 	}
    245 
    246 	if [ "$save_pwd" != `pwd` ]; then
    247 		tst_resm TWARN "libcgroup_subsys: cleanup_signal() is broken"
    248 		cd "$save_pwd"
    249 	fi
    250 	result="${save_result}"
    251 }
    252 
    253 function assert_cgroup_rwfile()
    254 {
    255 	local file="$1"
    256 	local descr="$2"
    257 	local rc=0
    258 
    259 	if [ ! -e "${file}" ]; then
    260 		tst_resm ${LIB_TEST_STATE} "$descr missing"
    261 		rc=1
    262 	fi
    263 
    264 	if [ ! -f "${file}" ]; then
    265 		tst_resm ${LIB_TEST_STATE} "$descr is not a regular file"
    266 		rc=2
    267 	fi
    268 
    269 	if [ ! -r "${file}" ]; then
    270 		tst_resm ${LIB_TEST_STATE} "$descr is not readable"
    271 		rc=3
    272 	fi
    273 
    274 	if [ ! -w "${file}" ]; then
    275 		tst_resm ${LIB_TEST_STATE} "$descr is not writeable"
    276 		rc=4
    277 	fi
    278 
    279 	[ $rc -ne 0 ] && {
    280 		result=${result:-$rc}
    281 		local s="$(stat "${file}")"
    282 		tst_resm ${LIB_TEST_STATE} "${s}"
    283 	}
    284 
    285 	return $rc
    286 }
    287 
    288 function assert_cgroup_tasks_rwfile()
    289 {
    290 	assert_cgroup_rwfile "tasks" "task list"
    291 	return $?
    292 }
    293 
    294 function dump_named_cgroup_tasks()
    295 {
    296 	local cgroup_name="$1"
    297 	local tasks
    298 
    299 	tasks=( $(cat "${cgroup_name}/tasks") ) # don't assign directly (bash bug)
    300 	if [ -z "${tasks[*]}" ]; then
    301 		return 0
    302 	fi
    303 	ps -p "${tasks[*]}" -o 'pid,ppid,pgid,tid,tpgid,blocked,ignored,pending,stat,tty,args'
    304 }
    305 
    306 function dump_cgroup_tasks()
    307 {
    308 	dump_named_cgroup_tasks "$(pwd)"
    309 }
    310 
    311 function assert_cgroup_tasks_empty()
    312 {
    313 	local nlines=$(( `cat tasks | wc -l` + 0))
    314 	local rc=$?
    315 
    316 	[ $rc -eq 0 -a $nlines -eq 0 ] && return 0
    317 	rc=$?
    318 	result=${result:-$rc}
    319 	tst_resm ${LIB_TEST_STATE} "cgroup task list is not empty: "
    320 	dump_cgroup_tasks 1>&2
    321 	return $rc
    322 }
    323 
    324 function assert_task_in_named_cgroup()
    325 {
    326 	local task_pid=$1
    327 	local cgroup_name="$2"
    328 
    329 	cat "${cgroup_name}/tasks" | grep -E "^${task_pid}\$" > /dev/null 2>&1 && return 0
    330 	local rc=$?
    331 	result=${result:-$rc}
    332 	tst_resm ${LIB_TEST_STATE} "Expected pid ${task_pid} is not in \"${cgroup_name}\" task list"
    333 	dump_named_cgroup_tasks "${cgroup_name}" 1>&2
    334 	return $rc
    335 }
    336 
    337 function assert_task_not_in_named_cgroup()
    338 {
    339 	local task_pid=$1
    340 	local cgroup_name="$2"
    341 
    342 	cat "${cgroup_name}/tasks" | grep -E "^${task_pid}\$" > /dev/null 2>&1 || return 0
    343 	local rc=1 # $? == 0 is an error in this case
    344 	result=${result:-$rc}
    345 	tst_resm ${LIB_TEST_STATE} "Expected pid ${task_pid} is in \"${cgroup_name}\" task list"
    346 	dump_named_cgroup_tasks "${cgroup_name}" 1>&2
    347 	return $rc
    348 }
    349 
    350 function assert_task_in_cgroup()
    351 {
    352 	assert_task_in_named_cgroup $1 "$(pwd)"
    353 	return $?
    354 }
    355 
    356 function assert_task_not_in_cgroup()
    357 {
    358 	assert_task_not_in_named_cgroup $1 "$(pwd)"
    359 	return $?
    360 }
    361 
    362 function assert_sample_proc_in_cgroup()
    363 {
    364 	assert_task_in_cgroup $sample_proc
    365 	return $?
    366 }
    367 
    368 function assert_sample_proc_not_in_cgroup()
    369 {
    370 	assert_task_not_in_cgroup $sample_proc
    371 	return $?
    372 }
    373 
    374 function assert_sample_proc_in_named_cgroup()
    375 {
    376 	assert_task_in_named_cgroup $sample_proc "$1"
    377 	return $?
    378 }
    379 
    380 function assert_sample_proc_not_in_named_cgroup()
    381 {
    382 	assert_task_not_in_named_cgroup $sample_proc "$1"
    383 	return $?
    384 }
    385 
    386 function get_task_state()
    387 {
    388 	ps -p $1 -o 'state=' 2>/dev/null
    389 }
    390 
    391 # TODO Check: Do these need to ignore case differences?
    392 function assert_task_state()
    393 {
    394 	local task_pid=$1
    395 	local expected_state="$2"
    396 	local ps_state="$(get_task_state ${task_pid})"
    397 	local rc=$?
    398 
    399 	[ $rc -eq 0 -a "$ps_state" == "${expected_state}" ] && return 0
    400 	rc=$?
    401 	result=${result:-$rc}
    402 	tst_resm ${LIB_TEST_STATE} "Expected task ${task_pid} to be in state \"${expected_state}\""
    403 	return $rc
    404 }
    405 
    406 #
    407 # Check that the specified task is not in the specified state
    408 #
    409 function assert_task_not_in_state()
    410 {
    411 	local task_pid=$1
    412 	local expected_state="$2"
    413 	local ps_state="$(get_task_state ${task_pid})"
    414  	local rc=$?
    415 
    416 	[ $rc -eq 0 -a "$ps_state" != "${expected_state}" ] && return 0
    417 	rc=$?
    418 	result=${result:-$rc}
    419 	tst_resm ${LIB_TEST_STATE} "Expected task ${task_pid} to not be in state \"${expected_state}\""
    420 	return $rc
    421 }
    422 
    423 #
    424 # Frozen tasks are in the "D" state according to ps
    425 # tasks in "T" state may also be in a "frozen" state
    426 #
    427 function assert_task_not_frozen()
    428 {
    429 	local task_pid=$1
    430 	local ps_state="$(ps -p $task_pid -o 'state=')"
    431 	local rc=$?
    432 
    433 	[ $rc -eq 0 -a "$ps_state" != "D" ] && return 0
    434 	rc=$?
    435 	result=${result:-$rc}
    436 	tst_resm ${LIB_TEST_STATE} "Expected task ${task_pid} is not frozen (unexpected task state: \"$ps_state\")"
    437 	return $rc
    438 }
    439 
    440 function assert_task_is_frozen()
    441 {
    442 	local task_pid=$1
    443 	local ps_state="$(ps -p $task_pid -o 'state=')"
    444 	local rc=$?
    445 
    446 	[ $rc -eq 0 -a "$ps_state" == "D" -o "$ps_state" == "T" ] && return 0
    447 	rc=$?
    448 	result=${result:-$rc}
    449 	tst_resm ${LIB_TEST_STATE} "Expected task ${task_pid} to be frozen (unexpected task state: \"$ps_state\")"
    450 	return $rc
    451 }
    452 
    453 function assert_sample_proc_not_frozen()
    454 {
    455 	assert_task_not_frozen $sample_proc
    456 	return $?
    457 }
    458 
    459 function assert_sample_proc_is_frozen()
    460 {
    461 	assert_task_is_frozen $sample_proc
    462 	return $?
    463 }
    464 
    465 function assert_sample_proc_stopped()
    466 {
    467 	assert_task_state $sample_proc 'T'
    468 	return $?
    469 }
    470 
    471 function assert_sample_proc_not_stopped()
    472 {
    473 	assert_task_not_in_state $sample_proc 'T'
    474 	return $?
    475 }
    476 
    477 function assert_sample_proc_sleeping()
    478 {
    479 	assert_task_state $sample_proc 'S'
    480 	return $?
    481 }
    482 
    483 function assert_sample_proc_not_sleeping()
    484 {
    485 	assert_task_not_in_state $sample_proc 'S'
    486 	return $?
    487 }
    488 
    489 function assert_cgroup_subsys_state_rwfile()
    490 {
    491 	if [ "${cgroup_subsys}" == "freezer" ]; then
    492 		assert_cgroup_rwfile "freezer.state" "freezer state"
    493 		return $?
    494 	elif [ "${cgroup_subsys}" == "freezer" ]; then
    495 		assert_cgroup_rwfile "signal.kill" "signal file"
    496 		return $?
    497 	else
    498 		return -1
    499 	fi
    500 }
    501 
    502 function get_freezer_state()
    503 {
    504 	local state="$(cat freezer.state)"
    505 	local rc=$?
    506 
    507 	if [ $rc -ne 0 ]; then
    508 		result=${result:-$rc}
    509 		tst_resm ${LIB_TEST_STATE} "Failed to read freezer state."
    510 		return $rc
    511 	fi
    512 	echo "${state}"
    513 	return 0
    514 }
    515 
    516 function assert_cgroup_freezer_state()
    517 {
    518 	local goal_state="$1"
    519 	local state="$(get_freezer_state)"
    520 	local rc=$?
    521 
    522 	[ $rc -eq 0 -a "${state}" == "${goal_state}" ] && return 0
    523 	rc=$?
    524 	result=${result:-$rc}
    525 	tst_resm ${LIB_TEST_STATE} "Expected freezer state \"$2\" but found freezer state: \"$state\""
    526 	return $rc
    527 }
    528 
    529 function make_sample_cgroup_named()
    530 {
    531 	local name="$1"
    532 	local saved_dir="$(pwd)"
    533 	mkdir "${name}"
    534 	local rc=$?
    535 
    536 	# So long as we made the directory we don't care
    537 	if [ ! -d "${name}" -a $rc -ne 0 ]; then
    538 		# But if it doesn't exist report the exit status of mkdir
    539 		result=${result:-$rc}
    540 		return $rc
    541 	fi
    542 
    543 	cd "${name}" > /dev/null 2>&1
    544 
    545 	rc=$?
    546 	if [ $rc -ne 0 ]; then
    547 		result=${result:-$rc}
    548 		return $rc
    549 	fi
    550 
    551 	assert_cgroup_tasks_rwfile || {
    552 		cd "${saved_dir}"
    553 		return $?
    554 	}
    555 	assert_cgroup_tasks_empty || {
    556 		cd "${saved_dir}"
    557 		return $?
    558 	}
    559 	assert_cgroup_subsys_state_rwfile || {
    560 		cd "${saved_dir}"
    561 		return $?
    562 	}
    563 	cd "${saved_dir}"
    564 	return 0
    565 }
    566 
    567 function make_sample_cgroup()
    568 {
    569 	make_sample_cgroup_named "child"
    570 	local rc=$?
    571 
    572 	# So long as we made the directory we don't care
    573 	if [ $rc -ne 0 ]; then
    574 		return $rc
    575 	fi
    576 
    577 	cd "child" # we know this will succeed since make_sample_cgroup_named
    578 		   # tested this
    579 	return 0
    580 }
    581 
    582 function rm_sample_cgroup_named()
    583 {
    584 	local cgroup_name="$1"
    585 	local saved_dir="$(pwd)"
    586 	local rc=0
    587 
    588 	cd "${cgroup_name}" && {
    589 		assert_cgroup_tasks_rwfile || {
    590 			cd "${saved_dir}"
    591 			return $?
    592 		}
    593 		assert_cgroup_tasks_empty || {
    594 			cd "${saved_dir}"
    595 			return $?
    596 		}
    597 		assert_cgroup_subsys_state_rwfile || {
    598 			cd "${saved_dir}"
    599 			return $?
    600 		}
    601 		cd "${saved_dir}"
    602 	} || {
    603 		rc=$?
    604 		result=${result:-$rc}
    605 		return $rc
    606 	}
    607 
    608 	[ -d "${cgroup_name}" ] && rmdir "${cgroup_name}" && return 0
    609 	rc=$?
    610 	tst_resm TWARN "Failed to remove cgroup \"${cgroup_name}\""
    611 	result=${result:-$rc}
    612 	return $rc
    613 }
    614 
    615 function rm_sample_cgroup()
    616 {
    617 	local cgroup_name="$(basename $(pwd))"
    618 	local rc=0
    619 
    620 	cd .. || {
    621 		rc=$?
    622 		result=${result:-$rc}
    623 		return $rc
    624 	}
    625 	rm_sample_cgroup_named "${cgroup_name}"
    626 	return $?
    627 }
    628 
    629 function ls_pids()
    630 {
    631 	ps -e -o 'pid=' | sed -e 's/[[:space:]]\+//g'
    632 }
    633 
    634 function assert_task_exists()
    635 {
    636 	local task_pid=$1
    637 
    638 	ls_pids | grep -E "^${task_pid}\$" > /dev/null 2>&1 && return 0
    639 	local rc=$?
    640 
    641 	result=${result:-$rc}
    642 	tst_resm ${LIB_TEST_STATE} "Expected pid ${task_pid} does not exist"
    643 	return $rc
    644 }
    645 
    646 function assert_task_does_not_exist()
    647 {
    648 	local task_pid=$1
    649 
    650 	ls_pids | grep -E "^${task_pid}\$" > /dev/null 2>&1 || return 0
    651 	local rc=1 # $? == 0 is an error in this case
    652 
    653 	result=${result:-$rc}
    654 	tst_resm ${LIB_TEST_STATE} "Did not expect pid ${task_pid} to exist"
    655 	return $rc
    656 }
    657 
    658 function assert_sample_proc_exists()
    659 {
    660 	assert_task_exists $sample_proc
    661 	return $?
    662 }
    663 
    664 function assert_sample_proc_does_not_exist()
    665 {
    666 	assert_task_does_not_exist $sample_proc
    667 	return $rc
    668 }
    669 
    670 function start_sample_proc()
    671 {
    672 	local sample_cmd="/bin/sleep"
    673 	local args
    674 
    675 	args=( $sample_sleep ) # can't assign directly because of bash v2/v3 inconsistency
    676 	if [ $# -gt 0 ]; then
    677 		sample_cmd="$1"
    678 		shift 1
    679 		args=( "$@" )
    680 	fi
    681 
    682 	[ -n "$sample_proc" ] && assert_sample_proc_does_not_exist
    683 
    684 	"$sample_cmd" "${args[@]}" &
    685 	local rc=$?
    686 	export sample_proc=$!
    687 	assert_sample_proc_exists
    688 
    689 	return $rc
    690 }
    691 
    692 function add_sample_proc_to_named_cgroup()
    693 {
    694 	local cgroup_name="$1"
    695 
    696 	assert_sample_proc_exists
    697 	"${CG_FILE_WRITE}" $sample_proc > "${cgroup_name}/tasks"
    698 	local rc=$?
    699 	if [ $rc -ne 0 ]; then
    700 		result=${result:-$rc}
    701 		tst_resm ${LIB_TEST_STATE} "Failed to add sample process $sample_proc to cgroup \"${cgroup_name}\""
    702 		return $rc
    703 	fi
    704 	assert_task_in_named_cgroup $sample_proc "${cgroup_name}"
    705 	return $?
    706 }
    707 
    708 function add_sample_proc_to_cgroup()
    709 {
    710 	add_sample_proc_to_named_cgroup "$(pwd)"
    711 	return $?
    712 }
    713 
    714 function kill_sample_proc()
    715 {
    716 	if [ -z "$sample_proc" ]; then
    717 		# It's no longer running or never started.
    718 		# If it was supposed to have started but did not then that
    719 		# should be determined by checking start_sample_proc results.
    720 		return 0
    721 	fi
    722 
    723 	# Hey, bash, don't print out any of your messy job status notices
    724 	disown -a
    725 
    726 	if [ "$(get_task_state $sample_proc)" == "D" ]; then
    727 		tst_resm TWARN "sample process is frozen stiff"
    728 		kill $sample_proc
    729 		local rc=$?
    730 		result=${result:-$rc}
    731 		return $rc
    732 	fi
    733 
    734 	# kill child processes of the sample process
    735 	while pgrep -P $sample_proc ; do
    736 		pkill -SIGTERM -P $sample_proc
    737 		sleep 1
    738 		pkill -SIGKILL -P $sample_proc
    739 
    740 		ps -p $(pgrep -P $sample_proc) -o 'state=' | grep -v D && continue
    741 		# Give up if all the child processes are frozen in D state or
    742 		# if there aren't any more child processes
    743 		break
    744 	done
    745 	# DEBUG dump pstree under $sample_proc:
    746 	# pstree -A -p $sample_proc
    747 	kill $sample_proc > /dev/null 2>&1 || kill -s SIGKILL $sample_proc > /dev/null 2>&1 || {
    748 		local rc=$?
    749 
    750 		ps -p $sample_proc -o 'state=' > /dev/null 2>&1
    751 		if [ $? -eq 1 ]; then
    752 			# It's dead. We're OK.
    753 			return 0
    754 		fi
    755 		# It's still alive somehow! Give up.
    756 		result=${result:-$rc}
    757 		tst_resm TWARN "Failed to kill sample process $sample_proc (kill exit status: $rc)"
    758 	}
    759 	assert_sample_proc_not_in_cgroup
    760 	assert_sample_proc_does_not_exist
    761 	return $?
    762 }
    763 
    764 function issue_freeze_cmd()
    765 {
    766 	local goal_state="FROZEN"
    767 	local sample_state_count=1
    768 	local state="$(get_freezer_state)"
    769 	local rc=$?
    770 
    771 	if [ $rc -ne 0 ]; then
    772 		return $rc
    773 	fi
    774 
    775 	while [ "${state}" != "${goal_state}" ]; do
    776 		"${CG_FILE_WRITE}" "${FREEZE}" > freezer.state
    777 		sleep $sample_state_period
    778 		state="$(get_freezer_state)"
    779 		rc=$?
    780 		if [ $rc -ne 0 ]; then
    781 			break
    782 		fi
    783 
    784 		((sample_state_count++))
    785 		if [ "$sample_state_count" -ge "$max_state_samples" ]; then
    786 			break
    787 		fi
    788 	done
    789 
    790 	if [ "${state}" == "${goal_state}" ]; then
    791 		return 0
    792 	fi
    793 
    794 	result=${result:-$rc}
    795 	tst_resm ${LIB_TEST_STATE} "Failed to issue freeze command (freezer state: \"`get_freezer_state`\")."
    796 	return $rc
    797 }
    798 
    799 # If we're trying to "freeze" tasks with SIGTOP
    800 function issue_stop_as_freeze_cmd()
    801 {
    802 	local goal_state="T"
    803 	local sample_state_count=1
    804 	local ps_state="$(get_task_state ${task_pid})"
    805 	local rc=$?
    806 
    807 	if [ $rc -ne 0 ]; then
    808 		return $rc
    809 	fi
    810 
    811 	while [ "${ps_state}" != "${goal_state}" ]; do
    812 		kill -s SIGSTOP $sample_proc
    813 		sleep $sample_state_period
    814 		ps_state="$(get_task_state ${task_pid})"
    815 		rc=$?
    816 		if [ $rc -ne 0 ]; then
    817 			break
    818 		fi
    819 
    820 		((sample_state_count++))
    821 		if [ "$sample_state_count" -ge "$max_state_samples" ]; then
    822 			break
    823 		fi
    824 	done
    825 
    826 	if [ "${ps_state}" == "${goal_state}" ]; then
    827 		return 0
    828 	fi
    829 
    830 	result=${result:-$rc}
    831 	tst_resm ${LIB_TEST_STATE} "Failed to issue stop (freeze) command (task state: \"${ps_state}\")."
    832 	return $rc
    833 }
    834 
    835 function send_signal()
    836 {
    837 	"${CG_FILE_WRITE}" $1 > 'signal.kill' && return 0
    838 	local rc=$?
    839 	result=${result:-$rc}
    840 	tst_resm ${LIB_TEST_STATE} "Failed to send signal: $1 to tasks in cgroup (rc: $rc)"
    841  	return $rc
    842 }
    843 
    844 function wait_until_goal_state_or_timeout()
    845 {
    846 	local goal_state="$1"
    847 	local sample_state_count=1
    848 	local state="$(get_freezer_state)"
    849 	local rc=$?
    850 
    851 	if [ $rc -ne 0 ]; then
    852 		return $rc
    853 	fi
    854 
    855 	while [ "${state}" != "${goal_state}" ]; do
    856 		sleep $sample_state_period
    857 		state="$(get_freezer_state)"
    858 		rc=$?
    859 		if [ $rc -ne 0 ]; then
    860 			break
    861 		fi
    862 
    863 		((sample_state_count++))
    864 		if [ "$sample_state_count" -ge "$max_state_samples" ]; then
    865 			break
    866 		fi
    867 	done
    868 	return $rc
    869 }
    870 
    871 # TODO convert signal scripts -- insert task between until and goal
    872 function wait_until_sample_proc_goal_state_or_timeout()
    873 {
    874 	local goal_state="$1"
    875 	local sample_state_count=1
    876 	local ps_state="$(get_task_state ${sample_proc})"
    877 	local rc=$?
    878 
    879 	while [ $rc -eq 0 -a "${ps_state}" != "${goal_state}" -a \
    880 		"$sample_state_count" -lt "$max_state_samples" ]; do
    881 		sleep $sample_state_period
    882 		ps_state="$(get_task_state ${sample_proc})"
    883 		rc=$?
    884 		if [ $rc -ne 0 ]; then
    885 			result=${result:-$rc}
    886 			tst_resm ${LIB_TEST_STATE} "Failed to read process state."
    887 			break
    888 		fi
    889 
    890 		((sample_state_count++))
    891 	done
    892 	return $rc
    893 }
    894 
    895 # TODO convert signal scripts -- insert task between until and not
    896 function wait_until_sample_proc_not_goal_state_or_timeout()
    897 {
    898 	local goal_state="$1"
    899 	local sample_state_count=1
    900 	local ps_state="$(get_task_state ${sample_proc})"
    901 	local rc=$?
    902 
    903 	while [ $rc -eq 0 -a "${ps_state}" == "${goal_state}" -a \
    904 		"$sample_state_count" -lt "$max_state_samples" ]; do
    905 		sleep $sample_state_period
    906 		ps_state="$(get_task_state ${sample_proc})"
    907 		rc=$?
    908 		if [ $rc -ne 0 ]; then
    909 			result=${result:-$rc}
    910 			tst_resm ${LIB_TEST_STATE} "Failed to read process state."
    911 			break
    912 		fi
    913 
    914 		((sample_state_count++))
    915 	done
    916 	return $rc
    917 }
    918 
    919 function wait_until_frozen()
    920 {
    921 	wait_until_goal_state_or_timeout "FROZEN" || return $?
    922 	assert_cgroup_freezer_state "FROZEN" "ERROR: failed to freeze cgroup"
    923 	# TODO assert all tasks in cgroup are in 'D' or 'T' state
    924 	# TODO assert that trying to add a task to cgroup results in EBUSY
    925 	return $?
    926 }
    927 
    928 function issue_thaw_cmd()
    929 {
    930 	"${CG_FILE_WRITE}" "${THAW}" > freezer.state && return 0
    931 	local rc=$?
    932 	result=${result:-$rc}
    933 	tst_resm ${LIB_TEST_STATE} "Failed to issue thaw command."
    934 	return $rc
    935 }
    936 
    937 function issue_cont_as_thaw_cmd()
    938 {
    939 	local goal_state="T"
    940 	local sample_state_count=1
    941 	local ps_state="$(get_task_state ${task_pid})"
    942 	local rc=$?
    943 
    944 	if [ $rc -ne 0 ]; then
    945 		return $rc
    946 	fi
    947 
    948 	while [ "${ps_state}" == "${goal_state}" ]; do
    949 		kill -s SIGCONT $sample_proc
    950 		sleep $sample_state_period
    951 		ps_state="$(get_task_state ${task_pid})"
    952 		rc=$?
    953 		if [ $rc -ne 0 ]; then
    954 			break
    955 		fi
    956 
    957 		((sample_state_count++))
    958 		if [ "$sample_state_count" -ge "$max_state_samples" ]; then
    959 			break
    960 		fi
    961 	done
    962 
    963 	if [ "${ps_state}" != "${goal_state}" ]; then
    964 		return 0
    965 	fi
    966 
    967 	result=${result:-$rc}
    968 	tst_resm ${LIB_TEST_STATE} "Failed to issue continue (thaw) command (task state: \"${ps_state}\")."
    969 	return $rc
    970 }
    971 
    972 function wait_until_thawed()
    973 {
    974 	wait_until_goal_state_or_timeout "THAWED" || return $?
    975 	assert_cgroup_freezer_state "THAWED" "ERROR: Failed to thaw cgroup."
    976 	return $?
    977 }
    978 
    979 function wait_until_freezing()
    980 {
    981 	wait_until_goal_state_or_timeout "FREEZING"
    982 	# Time critical -- we race with the kernel as it freezes tasks in the
    983 	# cgroup. So rather than assert "FREEZING" we just return
    984 	return $?
    985 }
    986 
    987 function wait_until_sample_proc_stopped()
    988 {
    989 	wait_until_sample_proc_state_or_timeout 'T' || return $?
    990 	assert_sample_proc_stopped
    991 	return $?
    992 }
    993 
    994 function wait_until_sample_proc_not_stopped()
    995 {
    996 	wait_until_sample_proc_not_goal_state_or_timeout 'T' || return $?
    997 	assert_sample_proc_not_stopped
    998 	return $?
    999 }
   1000