#!/bin/sh -e

##########################################################################
#   Script description:
#       Find and terminate stray processes, those running outside the
#       scheduler.
#       
#   History:
#   Date        Name        Modification
#   2014-08-27  Jason Bacon - UITS/CEAS - Facilitator,EMS 942Begin
##########################################################################

usage()
{
    printf "Usage: $0 username\n"
    exit 1
}


##########################################################################
#   Function description:
#       Pause until user presses return
##########################################################################

pause()
{
    local junk
    
    printf "Press return to continue..."
    read junk
}


##########################################################################
#   Main
##########################################################################

case $# in
0)
    user_name=$(whoami)
    if [ $user_name = root ]; then
	printf "Cannot run $0 for root.\n"
	exit 1
    fi
    ;;

1)
    user_name=$1
    ;;

*)
    usage
    ;;
esac

cat << EOM

Note:

Since it is difficult to distinguish between stray processes and processes
that are part of a legitimate job, detection of strays on nodes in use by
the same user for active jobs is not supported at this time.

Hence, it is possible, though unlikely, that some stray processes could be
overlooked.

EOM
pause

# Continue if a node is unresponsive
set +e

for node in `cluster-compute-nodes`; do
    printf "$node\r"

    case `auto-ostype` in
    FreeBSD)
	# -Y silences xauth warnings and should not pose a security risk here
	processes=`ssh -o ConnectTimeout=30 $node ps -U $user_name | \
	    awk '$1 != "PID" && $5 != "ps" && $5 != "awk" && \
		$5 != "sshd:" && $5 != "bash" && $5 != "tcsh" { print $4 }' | \
		sort | uniq`
	;;

    RHEL)
	processes=`ssh -o ConnectTimeout=30 $node ps -U $user_name | \
	    awk '$1 != "PID" && $4 != "ps" && $4 != "awk" && \
		$4 != "sshd" && $4 != "bash" && $4 != "tcsh" { print $4 }' | \
		sort | uniq`
	;;

    *)
	printf "$0: Not supported on $(auto-ostype).\n"
	exit 1
	;;
    
    esac
    if [ -n "$processes" ]; then
	
	# Check for valid jobs AFTER spotting processes, since a job
	# might start after the check.
	
	# squeue -o '%n' is not working.  Use short list and expand.
	host_list=`squeue -hu $user_name -o '%N'`
	if [ -n "$host_list" ]; then
	    host_list=`hostlist --expand $host_list`
	fi
    
	if echo $host_list | fgrep -q $node; then
	    printf "\nUser $user_name has jobs running on $node.  Check for strays manually.\n"
	else
	    printf "\nStray processes found on $node:\n$processes\n"
	    case `auto-ostype` in
	    FreeBSD)
		# -Y silences xauth warnings and should not pose a security risk here
		ssh -o ConnectTimeout=30 $node ps -u -U $user_name | \
		    awk '$1 != "USER" && $5 != "ps" && $5 != "awk" && \
			$5 != "sshd:" && $5 != "bash" && $5 != "tcsh" { print $0 }'
		;;
	    
	    RHEL)
		ssh -o ConnectTimeout=30 $node ps -u -U $user_name | \
		    awk '$1 != "USER" && $11 != "ps" && $4 != "awk" && \
			$11 != "sshd" && $11 != "bash" && $11 != "tcsh" { print $0 }'
		;;
	    
	    esac
	    printf "\nTerminate? [y]/n "
	    read terminate
	    if [ 0$terminate != 0n ]; then
		#ssh -t $node top
		for process in $processes; do
		    printf "Killing $process...\n"
		    ssh $node pkill -9 $process
		    #ssh $node killall $process
		done
		printf 'Running "top" on $node.  Verify that no strays remain.\n'
		printf "Press 'q' to quit top.\n"
		pause
		ssh -t $node top
	    fi
	fi
    fi
done
printf "\n"

