#!/usr/local/bin/cbsd
#v13.0.8
MYARG=""
MYOPTARG="cbsdfile cur_env cwd jname multiple quiet"
MYDESC="create jail/bhyve env from CBSDfile, vagrant-like behavior"
CBSDMODULE="bhyve,jail"
ADDHELP="\

${H3_COLOR}Description${N0_COLOR}:

In addition to dialog(1)-based utilities (jconstruct-tui/bconstruct-tui/xconstruct-tui)
and creating environments through arguments (jcreate/bcreate/xcreate) there is another
method for creating environments in CBSD, when the environment is described by a file.
This is similar to the Vagrant method, where the file stores all the enclosure
information in one place.

To create an environment with 'cbsd up' script, you must create a file named CBSDfile
and execute 'cbsd up' command in this directory.

Some parameters can be reassigned in the command line, e,g. one CBSDfile
to build an image of several versions.

By default, all the environments that are described in the CBSDfile will be created,
but you can regulate specific environments through the argument.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}cbsdfile${N0_COLOR} - alternative path to CBSDfile, could be relative to the
            working directory, e.g: '/tmp/CBSDfile'.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd up ver=13.0
 # cbsd up ver=12.2
 # cbsd up jail1 vm2

${H3_COLOR}See also${N0_COLOR}:

 cbsd destroy --help
 cbsd bconstruct-tui --help
 cbsd bconstruct --help
 cbsd bcreate --help
 cbsd jconstruct-tui --help
 cbsd jconstruct --help
 cbsd jcreate --help

"

EXTHELP="wf_cbsdfile"

. ${subrdir}/nc.subr
. ${strings}
cwd=
jname=
CLOUD_URL=
CLOUD_KEY=
. ${cbsdinit}

# flags for multiple CBSDfile env
[ -z "${multiple}" ] && multiple=0
[ -z "${cur_env}" ] && cur_env=1

# skip notice for subcommand with CBSDfile/API
export CBSDFILE_RECURSIVE=1

# init CBSDfile data
if [ -n "${cwd}" ]; then
	[ ! -r ${cwd}/CBSDfile ] && err 1 "${N1_COLOR}${CBSD_APP}: no such CBSDfile: ${N2_COLOR}${cwd}/CBSDfile${N0_COLOR}"
	cbsdfile="${cwd}/CBSDfile"
	cd ${cwd}
fi
. ${subrdir}/cbsdfile.subr
. ${subrdir}/time.subr

bhyve_ssh_wait()
{
	local _orig_options
	_orig_options=$( set +o )

	# disable errexit/xtrace settings for a while..
	set +o xtrace
	set +o errexit

	local _attempt_max=60 _i _x _y
	readconf cbsdfile.conf
	if [ -n "${bhyve_ssh_wait_timeout}" ]; then
		_attempt_max="${bhyve_ssh_wait_timeout}"
	fi

	printf "${N1_COLOR}waiting VM ssh (${_attempt_max}s): ${N2_COLOR}${jname}${N0_COLOR}"
	# wait for VM boot
	# Some OSes are unstable after the first connection, we will check three times for stability
	pass=0
	_y=2		# test for timeout exceed
	printf "${N1_COLOR}[" 1>&1
	for _i in $( ${SEQ_CMD} 1 ${_attempt_max} ); do
		#_x=$( timeout 10 cbsd bexec jname=${jname} date > /dev/null 2>&1 )
		_x=$( timeout 10 cbsd bexec jname=${jname} date > /dev/null 2>&1 )
		if [ $? -eq 0 ]; then
			pass=$(( pass + 1 ))
		else
			pass=0
		fi
		case ${pass} in
			0)
				printf "." 1>&2
				;;
			1)
				printf "o" 1>&2
				;;
			3)
				printf "O" 1>&2
				;;
		esac
		# when three times pass, we believe that the system is stable
		[ ${pass} -eq 3 ] && break
		_y=$(( _y + 1 ))
		# some ssh flop, wait for stabilize (todo: more retry support?)
		sleep 2
		# todo: delete only ip
		[ -z "${CLOUD_URL}" ] && ${ARP_CMD} -d -n ${ci_ip4_addr} > /dev/null 2>&1
		if [ ${_y} -eq ${_attempt_max} ]; then
			# restore old settings
			set ${_orig_options}
			return 1
		fi
	done
	${ECHO} ": ${N2_COLOR}${_y}${N1_COLOR}/${_attempt_max}]${N0_COLOR}" 1>&1

	${ECHO} "${N1_COLOR}bhyve ssh available${N0_COLOR}"
	# restore old settings
	set ${_orig_options}

	return 0
}

jail_ssh_wait()
{
	local _orig_options
	_orig_options=$( set +o )

	# disable errexit/xtrace settings for a while..
	set +o xtrace
	set +o errexit

	local _attempt_max=60 _i _x _y
	readconf cbsdfile.conf
	if [ -n "${jail_ssh_wait_timeout}" ]; then
		_attempt_max="${jail_ssh_wait_timeout}"
	fi

	printf "${N1_COLOR}waiting jail ssh (${_attempt_max}s): ${N2_COLOR}${jname}${N0_COLOR}"
	# wait for VM boot
	# Some OSes are unstable after the first connection, we will check three times for stability
	pass=0
	_y=2		# test for timeout exceed
	printf "${N1_COLOR}[" 1>&1
	for _i in $( ${SEQ_CMD} 1 ${_attempt_max} ); do
		#_x=$( timeout 10 cbsd jexec jname=${jname} date > /dev/null 2>&1 )
		_x=$( timeout 10 cbsd jexec jname=${jname} date > /dev/null 2>&1 )
		if [ $? -eq 0 ]; then
			pass=$(( pass + 1 ))
		else
			pass=0
		fi
		case ${pass} in
			0)
				printf "." 1>&2
				;;
			1)
				printf "o" 1>&2
				;;
			3)
				printf "O" 1>&2
				;;
		esac
		# when three times pass, we believe that the system is stable
		[ ${pass} -eq 3 ] && break
		_y=$(( _y + 1 ))
		# some ssh flop, wait for stabilize (todo: more retry support?)
		sleep 2
		# todo: delete only ip
		[ -z "${CLOUD_URL}" ] && ${ARP_CMD} -d -n ${ci_ip4_addr} > /dev/null 2>&1
		if [ ${_y} -eq ${_attempt_max} ]; then
			# restore old settings
			set ${_orig_options}
			return 1
		fi
	done
	${ECHO} ": ${N2_COLOR}${_y}${N1_COLOR}/${_attempt_max}]${N0_COLOR}" 1>&1

	${ECHO} "${N1_COLOR}jail ssh available${N0_COLOR}"
	# restore old settings
	set ${_orig_options}

	return 0
}

# testing/wip
run_jail_cloud()
{
	local CURL_CMD=$( which curl )
	local JQ_CMD=$( which jq )

	[ -z "${CURL_CMD}" ] && err 1 "${N1_COLOR}cloud up requires curl, please install: ${N2_COLOR}pkg install -y curl${N0_COLOR}"
	[ -z "${JQ_CMD}" ] && err 1 "${N1_COLOR}cloud up requires jq, please install: ${N2_COLOR}pkg install -y textproc/jq${N0_COLOR}"

	${ECHO} "${N1_COLOR}run image via: ${N2_COLOR}${CLOUD_URL}${N0_COLOR}" 1>&2
	[ -z "${ssh_wait}" ] && ssh_wait=0

	${CAT_CMD} > up-${jname}.json <<EOF
{
  "type": "jail",
  "host_hostname": "${host_hostname}",
  "imgsize": "${imgsize}",
  "ram": "${vm_ram}",
  "cpus": "${vm_cpus}",
  "img": "${img}",
  "pkglist": "${pkglist}",
  "extras": "${extras}",
  "recomendation": "${recomendation}",
  "pubkey": "${CLOUD_KEY}"
}
EOF

	${CAT_CMD} > deploy-${jname}.sh <<EOF
${CURL_CMD} --no-progress-meter -X POST -H "Content-Type: application/json" -d @up-${jname}.json ${CLOUD_URL}/api/v1/create/${jname} | ${JQ_CMD}
EOF
	${CHMOD_CMD} +x deploy-${jname}.sh
	./deploy-${jname}.sh
	# without debug
	${RM_CMD} -f ./deploy-${jname}.sh ./up-${jname}.json

	#if fn_exists postcreate_${jname} 2>/dev/null; then
		# we cannot execute the postcreate function
		# if we do not wait for ssh, force ssh_wait?
		#jail_ssh_wait=1
	#fi

	if [ ${ssh_wait} -eq 1 ]; then
		jail_ssh_wait
		ret=$?
	else
		ret=0
	fi
	if [ ${ret} -eq 0 ]; then
		#jail_${jname}		# re-read variables

		if fn_exists postcreate_${jname} 2>/dev/null; then
			${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}${N0_COLOR}" 1>&2

			# export some variables via H_CBSD_ (for scp,expose, and so on...)
			export H_CBSD_jname="${jname}"

			# cleanup jail when postcreate action failed
			trap "set +o xtrace; ${ECHO} \"${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${W1_COLOR}failed${N1_COLOR} : cleanup${N0_COLOR}\" 1>&2  ; jremove ${jname}" HUP INT ABRT BUS TERM EXIT
			set -o errexit
			set -o xtrace

			postcreate_${jname}

			set +o xtrace
			set +o errexit
			unset H_CBSD_jname
			trap "" HUP INT ABRT BUS TERM EXIT
			${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${H3_COLOR}success${N0_COLOR}" 1>&2
		else
			${ECHO} "${N1_COLOR}  == postcreate function not specified: postcreate_${N2_COLOR}${jname}${N0_COLOR}" 1>&2
		fi

		jname="${ojname}"
		# per-env postup func
		if fn_exists postup_${jname} 2>/dev/null; then
			${ECHO} "${N1_COLOR}execute postup_${jname}${N0_COLOR}"
			postup_${jname}
			jname="${ojname}"		# for corrupted/changed jname in func
		fi
	else
		[ ${ssh_wait} -eq 1 ] && err 1 "${N1_COLOR}ssh failed${N0_COLOR}"
	fi
	return 0
}


run_jail()
{
	local _val _ret MYOPTARG JAIL_ARGS i ojname
	[ -z "${jname}" ] && err 1 "${N1_COLOR}run_jail: empty jname${N0_COLOR}"

	# should be in sync with jcreate script
	MYOPTARG="jconf inter customskel fstablocal delpkglist removejconf pkglist jprofile zfs_snapsrc pkg_bootstrap autorestart runasap etcupdate_init quiet sysrc from"
	. ${distsharedir}/jail-arg
	[ "${racct}" = "1" ] && . ${distsharedir}/rctl.conf
	JAIL_ARGS="${JARG}"
	MYOPTARG="${MYOPTARG} ${JAIL_ARGS} ${RCTL} ${RCTL_EXTRA} forms"

	if [ -n "${CLOUD_URL}" ]; then
		# unset all
		ojname="${jname}"

		for i in ${JAIL_ARGS}; do
			unset ${i}
		done

		jname="${ojname}"
		jail_${jname}

		run_jail_cloud
		_ret=$?
		return ${_ret}
	fi

	. ${subrdir}/rcconf.subr
	[ $? -eq 0 ] && err 1 "${N1_COLOR}already exist: ${N2_COLOR}${jname}${N0_COLOR}"

	ojname="${jname}"

	# push old cbsd workdir
	ocbsd_workdir="${workdir}"

	# unset all
	for i in ${MYOPTARG}; do
		unset ${i}
	done

	jname="${ojname}"

	# defaults
	readconf jcreate.conf
	readconf cbsdfile.conf

	if fn_exists globals 2>/dev/null; then
		globals
	fi

	jail_${jname}

	JAIL_ARGS=

	# construct non-empty args/params
	for i in ${MYOPTARG} ${RCTL} ${RCTL_EXTRA}; do
		eval _val="\$$i"
		[ -z "${_val}" ] && continue
		if [ -z "${JAIL_ARGS}" ]; then
			JAIL_ARGS="${i}=\"${_val}\""
		else
			JAIL_ARGS="${JAIL_ARGS} ${i}=\"${_val}\""
		fi
	done

	# remote node support: experimental
	# todo: taskd/detached
	if [ -n "${node}" ]; then
		nodescp ${myworkdir}/CBSDfile ${node}:tmp/CBSDfile
		${ECHO} "${N1_COLOR}invoking command: ${N2_COLOR} rexe node=${node} tryoffline=1 /usr/local/bin/cbsd up cbsdfile=~tmp/CBSDfile${N0_COLOR}" | ${TR_CMD} -d \\t
		set -o errexit
		rexe node=${node} tryoffline=1 /usr/local/bin/cbsd up cbsdfile=tmp/CBSDfile
		set +o errexit
		exit 0
	fi

	[ -z "${cbsd_workdir}" ] && cbsd_workdir="${ocbsd_workdir}"

	# per-env preup func
	if fn_exists preup_${jname} 2>/dev/null; then
		${ECHO} "${N1_COLOR}execute preup_${jname}${N0_COLOR}"
		preup_${jname}
		jname="${ojname}"		# for corrupted/changed jname in func
		${ECHO} "${N1_COLOR}execute preup_${jname} done${N0_COLOR}"
	fi

	if [ -d ${myworkdir}/skel ]; then
		JAIL_ARGS="${JAIL_ARGS} customskel=${myworkdir}/skel"
	fi
	if [ -d ${myworkdir}/jails-system ]; then
		JAIL_ARGS="${JAIL_ARGS} jailsysskeldir=${myworkdir}/jails-system"
	fi

	${ECHO} "${N1_COLOR}invoking command: ${N2_COLOR} ${ENV_CMD} cbsd_workdir="${cbsd_workdir}" jcreate jname=${jname} ${N1_COLOR}${JAIL_ARGS} ${CBSDFILE_ARGS}${N0_COLOR}" | ${TR_CMD} -d \\t
	set -o errexit
	${ENV_CMD} cbsd_workdir="${cbsd_workdir}" jcreate jname=${jname} ${JAIL_ARGS} ${CBSDFILE_ARGS}
	set +o errexit

	. ${subrdir}/rcconf.subr
	jail_${jname}		# re-read variables

	if [ -n "${forms}" ]; then
		${ECHO} "${N1_COLOR}  == apply forms module: ${N2_COLOR}${forms}${N0_COLOR}" 1>&2
		forms jname="${jname}" module="${forms}" inter=0
		ret=$?
	fi

	if fn_exists postcreate_${jname} 2>/dev/null; then
		${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}${N0_COLOR}" 1>&2
		# cleanup jail when postcreate action failed
		trap "set +o xtrace; ${ECHO} \"${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${W1_COLOR}failed${N1_COLOR} : cleanup${N0_COLOR}\" 1>&2  ; jremove ${jname}" HUP INT ABRT BUS TERM EXIT

		# export some variables via H_CBSD_ (for scp,expose, and so on...)
		export H_CBSD_jname="${jname}"

		set -o errexit
		set -o xtrace

		postcreate_${jname}

		set +o xtrace
		set +o errexit
		trap "" HUP INT ABRT BUS TERM EXIT

		# export some variables via H_CBSD_
		unset H_CBSD_jname

		${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${H3_COLOR}success${N0_COLOR}" 1>&2
	else
		${ECHO} "${N1_COLOR}  == postcreate function not specified: postcreate_${N2_COLOR}${jname}${N0_COLOR}" 1>&2
	fi

	. ${subrdir}/rcconf.subr
	# per-env postup func
	if fn_exists postup_${jname} 2>/dev/null; then
		${ECHO} "${N1_COLOR}execute postup_${jname}${N0_COLOR}"
		postup_${jname}
		${ECHO} "${N1_COLOR}execute postup_${jname} done${N0_COLOR}"
	fi

	# restore old workdir
	cbsd_workdir="${ocbsd_workdir}"
}

# testing/wip
run_bhyve_cloud()
{
	local CURL_CMD=$( which curl )
	[ -z "${CURL_CMD}" ] && err 1 "${N1_COLOR}cloud up requires curl, please install: ${N2_COLOR}pkg install -y curl${N0_COLOR}"
	${ECHO} "${N1_COLOR}run image via: ${N2_COLOR}${CLOUD_URL}${N0_COLOR}"
	[ -z "${ssh_wait}" ] && ssh_wait=0

	${CAT_CMD} > up-${jname}.json <<EOF
{
  "type": "bhyve",
  "host_hostname": "${host_hostname}",
  "imgsize": "${imgsize}",
  "ram": "${vm_ram}",
  "cpus": "${vm_cpus}",
  "img": "${img}",
  "extras": "${extras}",
  "recomendation": "${recomendation}",
  "pubkey": "${CLOUD_KEY}"
}
EOF

	${CAT_CMD} > deploy-${jname}.sh <<EOF
${CURL_CMD} -X POST -H "Content-Type: application/json" -d @up-${jname}.json ${CLOUD_URL}/api/v1/create/${jname}
EOF
	${CHMOD_CMD} +x deploy-${jname}.sh
	./deploy-${jname}.sh
	# without debug
	${RM_CMD} -f ./deploy-${jname}.sh ./up-${jname}.json

	if [ ${ssh_wait} -eq 1 ]; then
		bhyve_ssh_wait
		ret=$?
	else
		ret=0
	fi
	if [ ${ret} -eq 0 ]; then
		#bhyve_${jname}		# re-read variables

		if fn_exists postcreate_${jname} 2>/dev/null; then
			${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}${N0_COLOR}" 1>&2

			# export some variables via H_CBSD_ (for scp,expose, and so on...)
			export H_CBSD_jname="${jname}"

			# cleanup jail when postcreate action failed
			trap "set +o xtrace; ${ECHO} \"${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${W1_COLOR}failed${N1_COLOR} : cleanup${N0_COLOR}\" 1>&2  ; bremove ${jname}" HUP INT ABRT BUS TERM EXIT
			set -o errexit
			set -o xtrace

			postcreate_${jname}

			set +o xtrace
			set +o errexit
			unset H_CBSD_jname
			trap "" HUP INT ABRT BUS TERM EXIT
			${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${H3_COLOR}success${N0_COLOR}" 1>&2
		else
			${ECHO} "${N1_COLOR}  == postcreate function not specified: postcreate_${N2_COLOR}${jname}${N0_COLOR}" 1>&2
		fi

		jname="${ojname}"
		# per-env postup func
		if fn_exists postup_${jname} 2>/dev/null; then
			${ECHO} "${N1_COLOR}execute postup_${jname}${N0_COLOR}"
			postup_${jname}
			jname="${ojname}"		# for corrupted/changed jname in func
		fi
	else
		[ ${ssh_wait} -eq 1 ] && err 1 "${N1_COLOR}ssh failed${N0_COLOR}"
	fi
	return 0
}

run_bhyve()
{
	local MYOPTARG BHYVE_ARGS ojname _ret

	[ -z "${jname}" ] && err 1 "${N1_COLOR}run_bhyve: empty jname${N0_COLOR}"

	# allow all bhyve settings
	BHYVE_ARGS="relative_path jname path data rcconf host_hostname ip4_addr astart nic_hwaddr nic_ratelimit zfs_snapsrc runasap interface rctl_nice emulator \
	imgsize imgtype vm_cpus vm_ram vm_os_type vm_efi iso_site iso_img register_iso_name register_iso_as vm_hostbridge bhyve_flags virtio_type vm_os_profile \
	swapsize vm_iso_path vm_guestfs vm_vnc_port bhyve_generate_acpi bhyve_wire_memory bhyve_rts_keeps_utc bhyve_force_msi_irq bhyve_x2apic_mode \
	bhyve_mptable_gen bhyve_ignore_msr_acc cd_vnc_wait bhyve_vnc_resolution bhyve_vnc_tcp_bind bhyve_vnc_vgaconf nic_driver vnc_password media_auto_eject \
	vm_cpu_topology debug_engine xhci cd_boot_firmware jailed chrooted on_poweroff on_reboot on_crash is_cloud ci_jname ci_fqdn ci_template ci_interface \
	ci_ip4_addr ci_gw4 ci_nameserver_address ci_nameserver_searchci_adjust_inteface_helper ci_user_add ci_user_pw_user ci_user_pw_root ci_user_pubkey uuid \
	ci_interface_mtu ci_interface2 ci_interface_mtu2 ci_ip4_addr2 ci_gw42"

	if [ -n "${CLOUD_URL}" ]; then
		# unset all
		ojname="${jname}"

		for i in ${BHYVE_ARGS}; do
			unset ${i}
		done

		jname="${ojname}"
		bhyve_${jname}

		run_bhyve_cloud
		_ret=$?
		return ${_ret}
	fi

	. ${subrdir}/rcconf.subr
	[ $? -eq 0 ] && err 1 "${N1_COLOR}already exist: ${N2_COLOR}${jname}${N0_COLOR}"

	ojname="${jname}"
	# should be in sync with bcreate script (except: jname)
	MYOPTARG="jconf inter removejconf"
	MYOPTARG="${MYOPTARG} ${BHYVE_ARGS}"

	# push old cbsd workdir
	ocbsd_workdir="${workdir}"

	# unset all
	for i in ${MYOPTARG}; do
		unset ${i}
	done

	jname="${ojname}"

	# defaults
	readconf bhyve-freebsd-default.conf
	readconf cbsdfile.conf

	if fn_exists globals 2>/dev/null; then
		globals
	fi

	bhyve_${jname}

	BHYVE_ARGS=

	# construct non-empty args/params
	for i in ${MYOPTARG}; do
		eval _val="\$$i"
		[ -z "${_val}" ] && continue
		if [ -z "${BHYVE_ARGS}" ]; then
			BHYVE_ARGS="${i}=\"${_val}\""
		else
			BHYVE_ARGS="${BHYVE_ARGS} ${i}=\"${_val}\""
		fi
	done

	[ -z "${cbsd_workdir}" ] && cbsd_workdir="${ocbsd_workdir}"

	# per-env preup func
	if fn_exists preup_${jname} 2>/dev/null; then
		${ECHO} "${N1_COLOR}execute preup_${jname}${N0_COLOR}"
		preup_${jname}
		jname="${ojname}"		# for corrupted/changed jname in func
		${ECHO} "${N1_COLOR}execute preup_${jname} done${N0_COLOR}"
	fi

	${ECHO} "${N1_COLOR}invoking command: ${N2_COLOR} ${ENV_CMD} cbsd_workdir="${cbsd_workdir}" bcreate ${N1_COLOR}${BHYVE_ARGS} ${CBSDFILE_ARGS}${N0_COLOR}" | ${TR_CMD} -d \\t
	set -o errexit
	${ENV_CMD} cbsd_workdir="${cbsd_workdir}" bcreate ${BHYVE_ARGS} ${CBSDFILE_ARGS}
	set +o errexit

	# runasap not atomic with bcreate to ignore bhyve_ssh_wait_timeout when/if fetch_iso work
	[ ${runasap} -eq 1 ] && ${ENV_CMD} cbsd_workdir="${cbsd_workdir}" bstart jname=${jname} quiet=1

	# restore old workdir
	cbsd_workdir="${ocbsd_workdir}"

	if [ ${ssh_wait} -eq 1 ]; then
		bhyve_ssh_wait
		ret=$?
	else
		ret=0
	fi
	if [ ${ret} -eq 0 ]; then
		#bhyve_${jname}		# re-read variables

		if fn_exists postcreate_${jname} 2>/dev/null; then
			${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}${N0_COLOR}" 1>&2

			# export some variables via H_CBSD_ (for scp,expose, and so on...)
			export H_CBSD_jname="${jname}"

			# cleanup bhyve when postcreate action failed
			trap "set +o xtrace; ${ECHO} \"${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${W1_COLOR}failed${N1_COLOR} : cleanup${N0_COLOR}\" 1>&2  ; bremove ${jname}" HUP INT ABRT BUS TERM EXIT
			set -o errexit
			set -o xtrace

			postcreate_${jname}

			set +o xtrace
			set +o errexit
			trap "" HUP INT ABRT BUS TERM EXIT
			unset H_CBSD_jname
			${ECHO} "${N1_COLOR}  == execute postcreate function for: ${N2_COLOR}${jname}: ${H3_COLOR}success${N0_COLOR}" 1>&2
		else
			${ECHO} "${N1_COLOR}  == postcreate function not specified: postcreate_${N2_COLOR}${jname}${N0_COLOR}" 1>&2
		fi

		jname="${ojname}"
		# per-env postup func
		if fn_exists postup_${jname} 2>/dev/null; then
			${ECHO} "${N1_COLOR}execute postup_${jname}${N0_COLOR}"
			postup_${jname}
			jname="${ojname}"		# for corrupted/changed jname in func
		fi

	else
		[ ${ssh_wait} -eq 1 ] && err 1 "${N1_COLOR}ssh failed${N0_COLOR}"
	fi
	return 0
}

# MAIN
export NOINTER=1
if [ ${num_env} -eq 1 ]; then
	st_time=$( ${DATE_CMD} +%s )

	if [ ${multiple} -gt 0 ]; then
		# sleep/delay a random amount of time between 2 and 10
		# to minimize the probability that a large number of machines 
		# will simultaneously attempt to up. except first (cur_env=1) env.
		if [ ${cur_env} -ne 1 ]; then
			rand_delay=$( ${AWK_CMD} -v min=2 -v max=10 'BEGIN{srand(); print int(min+rand()*(max-min+1))}' )
			sleep ${rand_delay}
		fi
	fi

	# protection from multiple env
	if [ ${multiple} -eq 0 ]; then
		# global preup
		if fn_exists preup 2>/dev/null; then
			${ECHO} "${N1_COLOR}execute global preup function${N0_COLOR}"
			preup
			${ECHO} "${N1_COLOR}execute global preup function done${N0_COLOR}"
		fi
	fi

	if [ ${jail_num} -eq 1 ]; then
		jname="${jail_list}"
		run_jail
	elif [ ${bhyve_num} -eq 1 ]; then
		jname="${bhyve_list}"
		run_bhyve
	fi
	end_time=$( ${DATE_CMD} +%s )
	diff_time=$(( end_time - st_time ))
	diff_time=$( displaytime ${diff_time} )

	# protection from multiple env
	if [ ${multiple} -eq 0 ]; then
		# global postup
		if fn_exists postup 2>/dev/null; then
			postup
		fi
	fi

	${ECHO} "${N1_COLOR}${CBSD_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}" 1>&2
	exit 0
fi

# multiple run area
. ${subrdir}/multiple.subr

${ECHO} "${N1_COLOR}Hint: Press ${N2_COLOR}'Ctrl+t'${N1_COLOR} to see last logfile line for active task${N0_COLOR}" 1>&2
task_owner="up_multiple"

task_id=
task_id_cur=

# global preup
if fn_exists preup 2>/dev/null; then
	${ECHO} "${N1_COLOR}execute global preup function${N0_COLOR}"
	preup
	${ECHO} "${N1_COLOR}execute global preup function done${N0_COLOR}"
fi

# spawn command for all jail
max_env=0
for i in ${jail_list} ${bhyve_list}; do
	max_env=$(( max_env + 1 ))
done

cur_env=1
for jname in ${jail_list} ${bhyve_list}; do
	# we must inherit CBSD_PWD via cwd= for cbsd-related function in postcreate_ action
	task_id_cur=$( task mode=new logfile=${tmpdir}/${task_owner}.${jname}.log.$$ client_id=${jname} autoflush=0 owner=${task_owner} ${ENV_CMD} NOCOLOR=${NOCOLOR} /usr/local/bin/cbsd up multiple=${max_env} cur_env=${cur_env} cbsdfile=${Makefile} jname=${jname} cwd=${CBSD_PWD} )
	task_id="${task_id} ${task_id_cur}"
	multiple_task_id_all=$( echo ${task_id} | ${TR_CMD} " " "," )
	cur_env=$(( cur_env + 1 ))
	sleep 1
done

st_time=$( ${DATE_CMD} +%s )
multiple_processing_spawn -o ${task_owner} -n "up"

# global postup
if fn_exists postup 2>/dev/null; then
	${ECHO} "${N1_COLOR}execute global postup function${N0_COLOR}"
	postup
	${ECHO} "${N1_COLOR}execute global postup function done${N0_COLOR}"
fi

end_time=$( ${DATE_CMD} +%s )
diff_time=$(( end_time - st_time ))
diff_time=$( displaytime ${diff_time} )
${ECHO} "${N1_COLOR}${CBSD_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"

exit 0
