1 #!/bin/sh 2 ############################################################################## 3 # # 4 # Copyright (c) International Business Machines Corp., 2007 # 5 # Sivakumar Chinnaiah, Sivakumar.C (at] in.ibm.com # 6 # Copyright (c) Linux Test Project, 2016 # 7 # # 8 # This program is free software: you can redistribute it and/or modify # 9 # it under the terms of the GNU General Public License as published by # 10 # the Free Software Foundation, either version 3 of the License, or # 11 # (at your option) any later version. # 12 # # 13 # This program is distributed in the hope that it will be useful, # 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of # 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # 16 # GNU General Public License for more details. # 17 # # 18 # You should have received a copy of the GNU General Public License # 19 # along with this program. If not, see <http://www.gnu.org/licenses/>. # 20 # # 21 ############################################################################## 22 # # 23 # Description: Test Basic functionality of numactl command. # 24 # Test #1: Verifies cpunodebind and membind # 25 # Test #2: Verifies preferred node bind for memory allocation # 26 # Test #3: Verifies share memory allocation on preferred node # 27 # Test #4: Verifies memory interleave on all nodes # 28 # Test #5: Verifies share memory interleave on all nodes # 29 # Test #6: Verifies physcpubind # 30 # Test #7: Verifies localalloc # 31 # Test #8: Verifies memhog # 32 # Test #9: Verifies numa_node_size api # 33 # Test #10:Verifies Migratepages # 34 # Test #11:Verifies hugepage alloacted on specified node # 35 # Test #12:Verifies THP memory allocated on preferred node # 36 # # 37 ############################################################################## 38 39 TST_CNT=12 40 TST_SETUP=setup 41 TST_TESTFUNC=test 42 TST_NEEDS_TMPDIR=1 43 TST_NEEDS_ROOT=1 44 TST_NEEDS_CMDS="numactl numastat awk" 45 46 . tst_test.sh 47 48 # 49 # Extracts the value of given numa node from the `numastat -p` output. 50 # 51 # $1 - Pid number. 52 # $2 - Node number. 53 # 54 extract_numastat_p() 55 { 56 local pid=$1 57 local node=$(($2 + 2)) 58 59 echo $(numastat -p $pid |awk '/^Total/ {print $'$node'}') 60 } 61 62 check_for_support_numa() 63 { 64 local pid=$1 65 66 local state=$(awk '{print $3}' /proc/$pid/stat) 67 68 if [ $state = 'T' ]; then 69 return 0 70 fi 71 72 return 1 73 } 74 75 setup() 76 { 77 export MB=$((1024*1024)) 78 export PAGE_SIZE=$(tst_getconf PAGESIZE) 79 export HPAGE_SIZE=$(awk '/Hugepagesize:/ {print $2}' /proc/meminfo) 80 81 total_nodes=0 82 83 nodes_list=$(numactl --show | grep nodebind | cut -d ':' -f 2) 84 for node in $nodes_list; do 85 total_nodes=$((total_nodes+1)) 86 done 87 88 tst_res TINFO "The system contains $total_nodes nodes: $nodes_list" 89 if [ $total_nodes -le 1 ]; then 90 tst_brk TCONF "your machine does not support numa policy 91 or your machine is not a NUMA machine" 92 fi 93 } 94 95 # Verification of memory allocated on a node 96 test1() 97 { 98 Mem_curr=0 99 100 for node in $nodes_list; do 101 numactl --cpunodebind=$node --membind=$node support_numa alloc_1MB & 102 pid=$! 103 104 TST_RETRY_FUNC "check_for_support_numa $pid" 0 105 106 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 107 if [ $(echo "$Mem_curr < $MB" | bc) -eq 1 ]; then 108 tst_res TFAIL \ 109 "NUMA memory allocated in node$node is less than expected" 110 kill -CONT $pid >/dev/null 2>&1 111 return 112 fi 113 114 kill -CONT $pid >/dev/null 2>&1 115 done 116 117 tst_res TPASS "NUMA local node and memory affinity" 118 } 119 120 # Verification of memory allocated on preferred node 121 test2() 122 { 123 Mem_curr=0 124 125 COUNTER=1 126 for node in $nodes_list; do 127 128 if [ $COUNTER -eq $total_nodes ]; then #wrap up for last node 129 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 130 else 131 # always next node is preferred node 132 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 133 fi 134 135 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_1MB & 136 pid=$! 137 138 TST_RETRY_FUNC "check_for_support_numa $pid" 0 139 140 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * $MB" |bc) 141 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 142 tst_res TFAIL \ 143 "NUMA memory allocated in node$Preferred_node is less than expected" 144 kill -CONT $pid >/dev/null 2>&1 145 return 146 fi 147 148 COUNTER=$((COUNTER+1)) 149 kill -CONT $pid >/dev/null 2>&1 150 done 151 152 tst_res TPASS "NUMA preferred node policy" 153 } 154 155 # Verification of share memory allocated on preferred node 156 test3() 157 { 158 Mem_curr=0 159 COUNTER=1 160 161 for node in $nodes_list; do 162 163 if [ $COUNTER -eq $total_nodes ] #wrap up for last node 164 then 165 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 166 else 167 # always next node is preferred node 168 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 169 fi 170 171 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_1MB_shared & 172 pid=$! 173 174 TST_RETRY_FUNC "check_for_support_numa $pid" 0 175 176 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * $MB" |bc) 177 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 178 tst_res TFAIL \ 179 "NUMA share memory allocated in node$Preferred_node is less than expected" 180 kill -CONT $pid >/dev/null 2>&1 181 return 182 fi 183 184 COUNTER=$((COUNTER+1)) 185 kill -CONT $pid >/dev/null 2>&1 186 done 187 188 tst_res TPASS "NUMA share memory allocated in preferred node" 189 } 190 191 # Verification of memory interleaved on all nodes 192 test4() 193 { 194 Mem_curr=0 195 # Memory will be allocated using round robin on nodes. 196 Exp_incr=$(echo "$MB / $total_nodes" |bc) 197 198 numactl --interleave=all support_numa alloc_1MB & 199 pid=$! 200 201 TST_RETRY_FUNC "check_for_support_numa $pid" 0 202 203 for node in $nodes_list; do 204 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 205 206 if [ $(echo "$Mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 207 tst_res TFAIL \ 208 "NUMA interleave memory allocated in node$node is less than expected" 209 kill -CONT $pid >/dev/null 2>&1 210 return 211 fi 212 done 213 214 kill -CONT $pid >/dev/null 2>&1 215 tst_res TPASS "NUMA interleave policy" 216 } 217 218 # Verification of shared memory interleaved on all nodes 219 test5() 220 { 221 Mem_curr=0 222 # Memory will be allocated using round robin on nodes. 223 Exp_incr=$(echo "$MB / $total_nodes" |bc) 224 225 numactl --interleave=all support_numa alloc_1MB_shared & 226 pid=$! 227 228 TST_RETRY_FUNC "check_for_support_numa $pid" 0 229 230 for node in $nodes_list; do 231 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 232 233 if [ $(echo "$Mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 234 tst_res TFAIL \ 235 "NUMA interleave share memory allocated in node$node is less than expected" 236 kill -CONT $pid >/dev/null 2>&1 237 return 238 fi 239 done 240 241 kill -CONT $pid >/dev/null 2>&1 242 243 tst_res TPASS "NUMA interleave policy on shared memory" 244 } 245 246 # Verification of physical cpu bind 247 test6() 248 { 249 no_of_cpus=0 #no. of cpu's exist 250 run_on_cpu=0 251 running_on_cpu=0 252 253 no_of_cpus=$(tst_ncpus) 254 # not sure whether cpu's can't be in odd number 255 run_on_cpu=$(($((no_of_cpus+1))/2)) 256 numactl --physcpubind=$run_on_cpu support_numa pause & #just waits for sigint 257 pid=$! 258 var=`awk '{ print $2 }' /proc/$pid/stat` 259 while [ $var = '(numactl)' ]; do 260 var=`awk '{ print $2 }' /proc/$pid/stat` 261 tst_sleep 100ms 262 done 263 # Warning !! 39 represents cpu number, on which process pid is currently running and 264 # this may change if Some more fields are added in the middle, may be in future 265 running_on_cpu=$(awk '{ print $39; }' /proc/$pid/stat) 266 if [ $running_on_cpu -ne $run_on_cpu ]; then 267 tst_res TFAIL \ 268 "Process running on cpu$running_on_cpu but expected to run on cpu$run_on_cpu" 269 ROD kill -INT $pid 270 return 271 fi 272 273 ROD kill -INT $pid 274 275 tst_res TPASS "NUMA phycpubind policy" 276 } 277 278 # Verification of local node allocation 279 test7() 280 { 281 Mem_curr=0 282 283 for node in $nodes_list; do 284 numactl --cpunodebind=$node --localalloc support_numa alloc_1MB & 285 pid=$! 286 287 TST_RETRY_FUNC "check_for_support_numa $pid" 0 288 289 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 290 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 291 tst_res TFAIL \ 292 "NUMA localnode memory allocated in node$node is less than expected" 293 kill -CONT $pid >/dev/null 2>&1 294 return 295 fi 296 297 kill -CONT $pid >/dev/null 2>&1 298 done 299 300 tst_res TPASS "NUMA local node allocation" 301 } 302 303 check_ltp_numa_test8_log() 304 { 305 grep -m1 -q '.' ltp_numa_test8.log 306 } 307 308 # Verification of memhog with interleave policy 309 test8() 310 { 311 Mem_curr=0 312 # Memory will be allocated using round robin on nodes. 313 Exp_incr=$(echo "$MB / $total_nodes" |bc) 314 315 numactl --interleave=all memhog -r1000000 1MB >ltp_numa_test8.log 2>&1 & 316 pid=$! 317 318 TST_RETRY_FUNC "check_ltp_numa_test8_log" 0 319 320 for node in $nodes_list; do 321 Mem_curr=$(echo "$(extract_numastat_p $pid $node) * $MB" |bc) 322 323 if [ $(echo "$Mem_curr < $Exp_incr" |bc ) -eq 1 ]; then 324 tst_res TFAIL \ 325 "NUMA interleave memhog in node$node is less than expected" 326 kill -KILL $pid >/dev/null 2>&1 327 return 328 fi 329 done 330 331 kill -KILL $pid >/dev/null 2>&1 332 tst_res TPASS "NUMA MEMHOG policy" 333 } 334 335 # Function: hardware cheking with numa_node_size api 336 # 337 # Description: - Returns the size of available nodes if success. 338 # 339 # Input: - o/p of numactl --hardware command which is expected in the format 340 # shown below 341 # available: 2 nodes (0-1) 342 # node 0 size: 7808 MB 343 # node 0 free: 7457 MB 344 # node 1 size: 5807 MB 345 # node 1 free: 5731 MB 346 # node distances: 347 # node 0 1 348 # 0: 10 20 349 # 1: 20 10 350 # 351 test9() 352 { 353 RC=0 354 355 numactl --hardware > gavail_nodes 356 RC=$(awk '{ if ( NR == 1 ) {print $1;} }' gavail_nodes) 357 if [ $RC = "available:" ]; then 358 RC=$(awk '{ if ( NR == 1 ) {print $3;} }' gavail_nodes) 359 if [ $RC = "nodes" ]; then 360 RC=$(awk '{ if ( NR == 1 ) {print $2;} }' gavail_nodes) 361 tst_res TPASS "NUMA policy on lib NUMA_NODE_SIZE API" 362 else 363 tst_res TFAIL "Failed with numa policy" 364 fi 365 else 366 tst_res TFAIL "Failed with numa policy" 367 fi 368 } 369 370 # Verification of migratepages 371 test10() 372 { 373 Mem_curr=0 374 COUNTER=1 375 376 for node in $nodes_list; do 377 378 if [ $COUNTER -eq $total_nodes ]; then 379 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 380 else 381 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 382 fi 383 384 numactl --preferred=$node support_numa alloc_1MB & 385 pid=$! 386 387 TST_RETRY_FUNC "check_for_support_numa $pid" 0 388 389 migratepages $pid $node $Preferred_node 390 391 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * $MB" |bc) 392 if [ $(echo "$Mem_curr < $MB" |bc ) -eq 1 ]; then 393 tst_res TFAIL \ 394 "NUMA migratepages is not working fine" 395 kill -CONT $pid >/dev/null 2>&1 396 return 397 fi 398 399 COUNTER=$((COUNTER+1)) 400 kill -CONT $pid >/dev/null 2>&1 401 done 402 403 tst_res TPASS "NUMA MIGRATEPAGES policy" 404 } 405 406 # Verification of hugepage memory allocated on a node 407 test11() 408 { 409 Mem_huge=0 410 Sys_node=/sys/devices/system/node 411 412 if [ ! -d "/sys/kernel/mm/hugepages/" ]; then 413 tst_res TCONF "hugepage is not supported" 414 return 415 fi 416 417 for node in $nodes_list; do 418 Ori_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages) 419 New_hpgs=$((Ori_hpgs + 1)) 420 echo $New_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 421 422 Chk_hpgs=$(cat ${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages) 423 if [ "$Chk_hpgs" -ne "$New_hpgs" ]; then 424 tst_res TCONF "hugepage is not enough to test" 425 return 426 fi 427 428 numactl --cpunodebind=$node --membind=$node support_numa alloc_1huge_page & 429 pid=$! 430 TST_RETRY_FUNC "check_for_support_numa $pid" 0 431 432 Mem_huge=$(echo $(numastat -p $pid |awk '/^Huge/ {print $'$((node+2))'}')) 433 Mem_huge=$((${Mem_huge%.*} * 1024)) 434 435 if [ "$Mem_huge" -lt "$HPAGE_SIZE" ]; then 436 tst_res TFAIL \ 437 "NUMA memory allocated in node$node is less than expected" 438 kill -CONT $pid >/dev/null 2>&1 439 echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 440 return 441 fi 442 443 kill -CONT $pid >/dev/null 2>&1 444 echo $Ori_hpgs >${Sys_node}/node${node}/hugepages/hugepages-${HPAGE_SIZE}kB/nr_hugepages 445 done 446 447 tst_res TPASS "NUMA local node hugepage memory allocated" 448 } 449 450 # Verification of THP memory allocated on preferred node 451 test12() 452 { 453 Mem_curr=0 454 455 if ! grep -q '\[always\]' /sys/kernel/mm/transparent_hugepage/enabled; then 456 tst_res TCONF "THP is not supported/enabled" 457 return 458 fi 459 460 COUNTER=1 461 for node in $nodes_list; do 462 463 if [ $COUNTER -eq $total_nodes ]; then #wrap up for last node 464 Preferred_node=$(echo $nodes_list | cut -d ' ' -f 1) 465 else 466 # always next node is preferred node 467 Preferred_node=$(echo $nodes_list | cut -d ' ' -f $((COUNTER+1))) 468 fi 469 470 numactl --cpunodebind=$node --preferred=$Preferred_node support_numa alloc_2HPSZ_THP & 471 pid=$! 472 473 TST_RETRY_FUNC "check_for_support_numa $pid" 0 474 475 Mem_curr=$(echo "$(extract_numastat_p $pid $Preferred_node) * 1024" |bc) 476 if [ $(echo "$Mem_curr < $HPAGE_SIZE * 2" |bc ) -eq 1 ]; then 477 tst_res TFAIL \ 478 "NUMA memory allocated in node$Preferred_node is less than expected" 479 kill -CONT $pid >/dev/null 2>&1 480 return 481 fi 482 483 COUNTER=$((COUNTER+1)) 484 kill -CONT $pid >/dev/null 2>&1 485 done 486 487 tst_res TPASS "NUMA preferred node policy verified with THP enabled" 488 } 489 490 tst_run 491
