#!/bin/sh -e

##########################################################################
#   Script description:
#       Resume updated compute nodes
#
#   Arguments:
#       Directory containing updated-nodes file from slurm-update-idle-nodes
#
#   History:
#   Date        Name        Modification
#   2015        Jason Bacon Begin
##########################################################################

usage()
{
    printf "Usage: $0 log-directory\n"
    exit 1
}


##########################################################################
#   Main
##########################################################################

if [ $# != 1 ]; then
    usage
fi

dir="$1"

# FIXME: This does not limit resume to updated nodes, but resumes
# any node that was unexpectedly rebooted.
# Reason set when draining is overwritten at reboot
# sinfo -h -o '%n %T'|grep compute-005
down_nodes=`sinfo -h -o '%n %T' | awk '$2 ~ "down" { print $1 }' | sort | uniq`
drained_nodes=`sinfo -h -o '%n %T' | awk '$2 ~ "drained" { print $1 }' | sort | uniq`

status=0
for node in $down_nodes $drained_nodes; do
    if fgrep -q $node "$dir/updated-nodes"; then
	reason="`sinfo -h -n $node -o '%E'`"
	if [ "$reason" = slurm-update-idle-nodes ] || \
	    [ "$reason" = "Node unexpectedly rebooted" ]; then
	    if ssh -o ConnectTimeout=2 $node ls > /dev/null 2>&1; then
		printf "Resuming $node, down for $reason...\n"
		# Scontrol reports invalid node state with resume sometimes.  This
		# seems to fix it.
		scontrol update state=resume nodename=$node
	    else
		printf "$node is not responding.  May still be rebooting...\n"
		status=1
	    fi
	else
	    printf "Skipping $node, down for $reason\n"
	fi
    fi
done
exit $status
