#!/bin/sh

##########################################################################
#   Script description:
#       Various Unix tools for monitoring nodes
#       
#   History:
#   Date        Name        Modification
#   2014-01-08  J Bacon     Begin
##########################################################################

usage()
{
    printf "Usage: $0\n"
    exit 1
}


##########################################################################
#   Function description:
#       Pause until user presses return
##########################################################################

pause()
{
    local junk
    
    printf "Press return to continue..."
    read junk
}


##########################################################################
#   Function description:
#       List network interfaces on a node
#       
#   History:
#   Date        Name        Modification
#   2020-02-11  J Bacon     Begin
##########################################################################

list_interfaces()
{
    case $(auto-ostype) in
    FreeBSD)
	ssh $node ifconfig | fgrep UP
	;;
    
    RHEL)
	ssh $node ip link | fgrep UP
	;;

    *)
	printf "$0: Not supported on $os_name.\n"
	pause
	exit 1
	;;

    esac
}


##########################################################################
#   Main
##########################################################################

if [ $# != 0 ]; then
    usage
fi

while true; do

    clear
    spcm-banner
    cat << EOM
    
1.. Check node status
2.. Monitor node with top
3.. Monitor node with netstat
4.. Monitor node with iostat
5.. Monitor node with iftop
6.. Show cores per user
7.. Show jobs and cluster load
8.. Scan for stray processes
9.. Hardware specifications
Q.. Quit / Return to main menu

EOM

    printf "Selection? "
    read selection
    clear
    case $selection in
    1)
	cluster-node-status
	;;
    
    2)
	spcm-banner
	printf "\nHost name or compute-node number? "
	read node
	node-top $node
	;;
    
    3)
	spcm-banner
	printf "\nHost name or compute-node number? "
	read node
	node-netstat $node
	;;
    
    4)
	spcm-banner
	printf "\nHostname or compute-node number? "
	read node
	node-iostat $node
	;;
    
    5)
	spcm-banner
	printf "\nHostname or compute-node number? "
	read node
	if [ 0$(node-type compute-$node) = 0compute ]; then
	    node=compute-$node
	fi
	list_interfaces $node
	printf "Network interface? "
	read iface
	if [ 0$iface = 0 ]; then
	    node-iftop $node
	else
	    node-iftop $node -i $iface
	fi
	;;
    
    6)
	slurm-user-cores
	;;
    
    7)
	slurm-cluster-load
	;;
    
    8)
	spcm-banner
	printf "\nUsername of user with suspected strays? "
	read username
	slurm-find-strays $username
	;;

    9)
	cluster-hardware-specs 2>&1 | more
	;;
    
    Q|q)
	exit 0
	;;
    
    *)
	printf "Invalid selection: '$selection'\n"
	;;
    
    esac
    pause
done
