#!/bin/sh

##########################################################################
#   Script description:
#       Submenu for node management tasks
#       
#   History:
#   Date        Name        Modification
#   2015-12-19  J Bacon     Begin
##########################################################################

usage()
{
    printf "Usage: $0 \n"
    exit 1
}


##########################################################################
#   Function description:
#       Pause until user presses return
##########################################################################

pause()
{
    local junk
    
    read -p "Press return to continue..." junk
}


##########################################################################
#   Function description:
#       Collect info on hosts to be managed
#       
#   History:
#   Date        Name        Modification
#   2015-12-23  J Bacon     Begin
##########################################################################

get_host_info()
{
    default_digits=$1
    read -p "Digits in hostname #? [$default_digits] " host_digits
    if [ 0$host_digits = 0 ]; then
	host_digits=$default_digits
    fi
    
    # Remove leading zeros so printf doesn't interpret it as octal
    read -p "First host number? " first_num
    if [ 0$first_num = 0 ]; then
	return 1
    fi
    first_num=`echo $first_num | sed -E 's|^0+||'`
    read -p "Last host number? " last_num
    if [ 0$last_num = 0 ]; then
	return 1
    fi
    last_num=`echo $last_num | sed -E 's|^0+||'`
    return 0
}


##########################################################################
#   Function description:
#       
#   Arguments:
#       
#   Returns:
#       
#   History:
#   Date        Name        Modification
#   2016-10-27  root        Begin
##########################################################################

update_warning()
{
    cat << EOM

================================ WARNING ===================================

Rebooting the head node while slurm-update-idle-nodes is running may leave
a compute node partially updated and orphan

    $updates_dir/lockfile

It is best to update and reboot the head node before running
slurm-update-idle-nodes unless you really know what you are doing.

If you have already started updating idle compute nodes, it would be best
to allow *ALL* updates to finish before updating the head node.

Other sysadmin tasks such as new user addition are likely to fail while
cron updates are active, as nodes may reboot at any time.  Wait until most
nodes are updated before performing any sysadmin tasks that require access
to all compute nodes and be prepared to correct potential failures.

============================================================================
EOM
    pause
}

##########################################################################
#   Function description:
#       
#   Arguments:
#       
#   Returns:
#       
#   History:
#   Date        Name        Modification
#   2019-01-24  root        Begin
##########################################################################

get_node_type()
{
    node_type=none
    while [ $node_type = none ]; do
	cat << EOM

1.. Backup head nodes
2.. File servers
3.. Compute nodes
4.. Visualization nodes

EOM
	read node_type
	case $node_type in
	1)
	    node_type=head
	    default_digits=1
	    default_prefix=login-
	    ;;
	2)
	    node_type=io
	    default_digits=2
	    default_prefix=raid-
	    ;;
	3)
	    node_type=compute
	    default_digits=3
	    default_prefix=compute-
	    ;;
	4)
	    node_type=vis
	    default_digits=1
	    default_prefix=vis-
	    ;;
	*)
	    printf "Invalid selection.\n"
	    return 1
	esac
    done
    echo $node_type
}


default_responses_info()
{
    cat << EOM

At least one node must be configured before using default responses to
all prompts.

$LOCALBASE/etc/spcm should also contain fstab fragments for all NFS mounts.

EOM
}


##########################################################################
#   Main
##########################################################################

if [ $# != 0 ]; then
    usage
fi

LOCALBASE=$(cluster-localbase)

sinfo_format='%T %u %E %N'
updates_dir="/home/slurm/Compute-node-updates"

# FIXME: Is default_domain used by another script called from here?
if [ -e $LOCALBASE/etc/slurm.conf ]; then
    default_domain=`awk -F = '$1 == "ClusterName" { print $2 }' $LOCALBASE/etc/slurm.conf`
fi

while true; do

    clear
    spcm-banner
    cat << EOM
    
Installing updates remotely is not recommended.  Servers occasionally fail to
reboot, so you should be physically nearby in case of issues.

Updates are best performed in the order presented below.  Most importantly,
complete head node updates before beginning compute node updates.  Rebooting
the head node while compute node updates are in progress may leave a compute
node in a partially updated state.

1.. Install updates on head node
2.. Check for down nodes
3.. Install updates on visualization nodes
4.. Install updates on file servers
5.. Install updates on idle compute nodes
6.. Restart cron updates on idle compute nodes
7.. Upgrade nodes to a new OS release
Q.. Quit / Return to main menu

EOM

    read -p "Selection? " selection
    clear
    case $selection in
    
#    1)  # Update backup head node
#        cluster-update-backup-nodes
#        ;;
#    
#    2)  # Verify scheduler running on backup head node
#
#        if [ -z $(cluster-backup-nodes) ]; then
#            printf "No backup nodes found.\n"
#        else
#            ssh login-2 tail /var/log/slurm/slurmctld.log
#        fi
#        ;;
    
    1)  # Update head node
	# FIXME: Warn if cron updates are running
	update_warning
	if cluster-check-cron-updates; then
	    w
	    read -p 'Continue? y/[n] ' continue
	    if [ 0$continue = 0y ]; then
		    cat << EOM | wall

The head node may need to be rebooted shortly to complete the installation of
important security updates.  An additional notice will follow if a reboot
is necessary.

EOM
		auto-update-system --binary
		read -p "Reboot? [y]/n " reboot
		if [ `auto-ostype` = FreeBSD ]; then
		    read -p "Update and sync ports tree to all nodes? [y]/n " ports_update
		    if [ 0$ports_update != 0n ]; then
			cluster-ports-update
		    fi
		fi
		w
		if [ 0$reboot != 0n ]; then
		    read -p "Minutes before reboot? [2] " minutes
		    if [ 0$minutes = 0 ]; then
			minutes=2
		    fi
		    cat << EOM | wall

The head node will be rebooted shortly to complete the installation of
important security updates.  It should be available again in a few minutes.

EOM
		    shutdown -r +$minutes
		fi
	    fi
	fi
	;;
    
    2)
	cluster-node-status
	;;
    
    3)
	cluster-update-vis-nodes
	;;
    
    4)
	cluster-update-file-servers
	;;
    
    5)  # Update all compute nodes
	# FIXME: Warn if cron updates are running
	update_warning
	cat << EOM

You can update all compute nodes, or just those that have not received the
last round of updates.

EOM
	mkdir -p $updates_dir
	read -p "Update all compute nodes? y/[n] " update_all
	if [ 0$update_all = 0y ]; then
	    rm -f $updates_dir/updated-nodes
	fi
	slurm-update-idle-nodes $updates_dir
	;;
    
    6) # Schedule compute node updates in cron
	clear
	update_warning
	cat << EOM

This will clear the updated-nodes list from the last round of cron updates.

All compute nodes will be immediately set to draining state to prevent them
from receiving any new jobs before they are updated.

Be sure all head nodes and file servers are up before proceeding.

Nodes will be updated one at a time shortly after they become idle.

Note: cron must be configured with an entry like the following:

    0-59/15 * * * * slurm-update-idle-nodes --yes \\
	    >> /var/log/compute-node-updates 2>&1

slurm-update-idle-nodes and scripts it depends on must be in PATH.

EOM
	read -p "Proceed? y/[n] " proceed
	if [ 0"$proceed" = 0y ]; then
	    rm -f $updates_dir/updated-nodes $updates_dir/updates.log
	fi
	;;

    7)
	this_release=$(uname -r | cut -d - -f 1)
	major=${this_release%.*}
	minor=${this_release#*.}
	next_release=$major.$(($minor + 1))-RELEASE
	printf "Release? [$next_release] "
	read release
	if [ -z "$release" ]; then
	    release=$next_release
	fi
	cluster-upgrade $release
	;;
    
    Q|q)
	exit 0
	;;
    
    *)
	printf "Invalid selection: '$selection'\n"
	;;
    esac
    pause
done
