#!/bin/sh -e

##########################################################################
#   Script description:
#       Upgrade the OS on a node
#       
#   History:
#   Date        Name        Modification
#   2020-12-29  J Bacon     Begin
##########################################################################

usage()
{
    printf "Usage: $0 hostname version\n"
    printf "Example: $0 compute-001 12.2-RELEASE\n"
    exit 1
}


##########################################################################
#   Function description:
#       Wait for node to reboot
#       
#   History:
#   Date        Name        Modification
#   2020-12-29  Charlie &   Begin
##########################################################################

wait_for_reboot()
{
    local node=$1
    
    printf "Waiting for $node to reboot...\n"
    sleep 60
    while ! ssh $node uptime; do
	printf "Waiting for $node to reboot...\n"
	sleep 60
    done
}


##########################################################################
#   Function description:
#       Reboot a node if needed
#       
#   History:
#   Date        Name        Modification
#   2020-12-29  Charlie &   Begin
##########################################################################

reboot_node()
{
    local node=$1
    local reboot=''
    while [ 0$reboot != 0y ] && [ 0$reboot != 0n ]; do
	read -p "Reboot $node? y/n " reboot
    done
    if [ 0$reboot = 0y ]; then
	ssh $node shutdown -r now
	wait_for_reboot $node
    fi
}


##########################################################################
#   Main
##########################################################################

if [ $# != 2 ]; then
    usage
fi
node=$1
version=$2

case $(auto-ostype) in
FreeBSD)
    ssh $node node-type
    if [ $(node-type $node) = compute ]; then
	scontrol update nodename=$node state=drain reason="OS upgrade"
	sinfo --list-reasons
	state=`sinfo -n $node -o %T -h`
	if [ $state != drained ]; then
	    printf "$node is busy.  Try again later.\n"
	    exit 1
	fi
    fi
    
    printf "Updating current release before upgrade...\n"
    ssh -t $node freebsd-update fetch install || true
    reboot_node $node
    
    printf "Upgrading to $version...\n"
    ssh -t $node freebsd-update upgrade -r $version
    install=''
    while [ 0$install != 0y ] && [ 0$install != 0n ]; do
	read -p "Proceed with install on $node? y/n " install
    done
    if [ $install != y ]; then
	exit 1
    fi
    ssh -t $node freebsd-update install
    
    cat << EOM
    
First install phase complete.

Press return when ready to reboot $node.
    
EOM
    read junk
    ssh $node shutdown -r now || true
    wait_for_reboot $node
    
    printf "Running freebsd-update install on $node after reboot.\n"
    # Restart sshd in case API changes broke running daemon
    ssh -t $node "freebsd-update install && service sshd restart"
    
    if [ $(node-type $node) = compute ]; then
	resume=''
	while [ 0$resume != 0y ] && [ 0$resume != 0n ]; do
	    read -p "Resume $node? y/n " resume
	done
	if [ $resume = y ]; then
	    scontrol update nodename=$node state=resume
	fi
    fi
    
    printf "Rebooting again to reload kernel modules and restart services...\n"
    ssh $node shutdown -r now
    ;;

RHEL)
    printf "$0: Not supported on $(auto-ostype)\n"
    printf "Point-release upgrades are provided by Yum updates.\n"
    exit 1
    ;;

*)
    printf "$0: Not supported on $(auto-ostype)\n"
    exit 1
    ;;

esac
