#!/usr/local/bin/cbsd
#v12.1.8
MYARG=""
MYOPTARG="delay jname inter quiet"
MYDESC="Start jail"
ADDHELP="

${H3_COLOR}Description${N0_COLOR}:

 Start the container.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}delay=${N0_COLOR} - <sec>, delay N secbefore start, mainly to smooth the astart,
          default is: '0', no delay.
 ${N2_COLOR}inter=${N0_COLOR} - set 1 to prevent any questions and to accept answers by default.
 ${N2_COLOR}jname=${N0_COLOR} - target jail. If jail='*' or jail='pri*' then start all jails or
          jails whose names begin with 'pri', e.g. 'prison1', 'prisonX'...
 ${N2_COLOR}quiet=${N0_COLOR} - 0,1: be quiet, dont output verbose message.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd jstart
 # cbsd jstart jname='memcach*'

${H3_COLOR}See also${N0_COLOR}:

 cbsd jstop --help

"

CBSDMODULE="jail"
EXTHELP="wf_jstop_jstart"

. ${subrdir}/nc.subr
. ${tools}		# for select_jail_by_list

readconf buildworld.conf
readconf jail-freebsd-default.conf

if [ -n "${ci_gw4}" -a "${ci_gw4}" != "0" ]; then
	oci_gw4="${ci_gw4}"
else
	if [ -n "${default_ci_gw4}" -a "${default_ci_gw4}" != "0" ]; then
		oci_gw4="${default_ci_gw4}"
	fi
fi

[ -z "${1}" ] && select_jail_by_list -s "List of offline jail" -a "Off" -r ${sqlreplica}

delay=0
. ${cbsdinit}

. ${system}
. ${subrdir}/universe.subr
. ${subrdir}/ipfw.subr	# fwcounter

[ -z "${quiet}" ] && quiet=0

# trim args from "$*"
ARGS=
for i in $*; do
	prefix6=$( substr --pos=0 --len=6 --str="${i}" )

	[ "${prefix6}" = "delay=" ] && shift && continue
	[ "${prefix6}" = "jname=" ] && shift && continue
	[ "${prefix6}" = "quiet=" ] && shift && continue
	[ "${prefix6}" = "inter=" ] && shift && continue

	if [ -n "${ARGS}" ]; then
		ARGS="${ARGS} ${i}"
	else
		ARGS="${i}"
	fi
done

[ -z "${jname}" -a -z "${ARGS}" ] && err 1 "${N1_COLOR}No jail specified${N0_COLOR}"

# for external_exec-related command
. ${subrdir}/jcreate.subr

my_name="jstart"

# check environment scripts from tests dir
# when error code != 0 - exit
# $1 - is source dir name ( tests )
check_environment_script()
{
	local _res
	local _dir="${1}"
	local _testfiles

	[ -z "${_dir}" ] && return 1

	local _srcdir="${jailsysdir}/${jname}/${_dir}"

	[ ! -d "${_srcdir}" ] && return 0
	[ -z "$( ${LS_CMD} ${_srcdir}/ )" ] && return 0

	_testfiles=$( ${FIND_CMD} "${_srcdir}" -type l -or -type f -depth 1 -maxdepth 1 -exec ${BASENAME_CMD} {} \; )

	for _file in ${_testfiles}; do
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Check environment script: ${N2_COLOR}${_file}${N0_COLOR}"
		env jname=${jname} ${_srcdir}/${_file}
		[ $? -ne 0 ] && err 1 "${N1_COLOR}Error: ${N2_COLOR}${_file}${N0_COLOR}"
	done
}

exec_cbsdjail_first_boot()
{
	if [ -f ${path}/etc/rc.cbsdjail_first_boot ]; then
		local CBSDPATH="${PATH}"
		# reset CBSD PATH
		export PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
		/usr/local/bin/cbsd jexec jname=${jname} /bin/sh /etc/rc.cbsdjail_first_boot
		# restore CBSD PATH
		export PATH="${CBSDPATH}"
		${RM_CMD} -f ${path}/etc/rc.cbsdjail_first_boot
	fi
}

# prepare symlink for emulator and
# check that the emulator is able to execute commands
emulator_exec_check()
{
	local _res _ret _path

	[ -z "${emulator}" ] && return 0
	[ "${emulator}" = "bhyve" ] && return 0
	[ "${emulator}" = "jail" ] && return 0
	[ "${emulator}" = "0" ] && return 0

	if [ ${baserw} -eq 0 ]; then
		_path="${BASE_DIR}"
	else
		_path="${data}"
	fi

	preparebase dst=${_path} emulator=${emulator}
	if [ ! -r ${_path}/bin/${emulator} ]; then
		printf "no such ${_path}/bin/${emulator}, please check: cbsd preparebase dst=${_path} emulator=${emulator}"
		return 1
	fi
	_res=$( ${CHROOT_CMD} ${_path} /bin/${emulator} /bin/sh -c "echo ping" 2>&1 )
	_ret=$?
	[ ${_ret} -ne 0 ] && printf "${CHROOT_CMD} ${_path} /bin/${emulator} /bin/sh -c \"echo ping\": ${_res}"
	return ${_ret}
}

etcupdate_check()
{
	[ "${etcupdate_init}" != "1" ] && return 0
	[ "${empty}" = "1" ] && return 0

	local _etcupdate_jail_root

	_etcupdate_jail_root="${jailsysdir}/${jname}/etcupdate"
	[ ! -d ${_etcupdate_jail_root}/current ] && return 0		# not etcupdate bootstraped jail
	[ ! -r ${_etcupdate_jail_root}/ver ] && return 0			# no bootstrap version info
	. ${_etcupdate_jail_root}/ver
	[ -z "${etcupdate_current_ver}" ] && return 0			# no bootstrap version info

	if [ "${etcupdate_current_ver}" != "${ver}" ]; then
		[ ${quiet} -ne 1 ] && ${ECHO} "${W1_COLOR}Warning${N1_COLOR}: jail version (${N2_COLOR}${ver}${N1_COLOR}) and distribution (e.g: /etc dir content) (${N2_COLOR}${etcupdate_current_ver}${N1_COLOR}) is differ${N0_COLOR}"
		[ ${quiet} -ne 1 ] && ${ECHO} "${W1_COLOR}Warning${N1_COLOR}: you can use etcupdate to sync content: ${N2_COLOR}cbsd etcupdate jname=${jname} mode=update from=${etcupdate_current_ver} to=${ver}${N0_COLOR}"
	fi
	return 0
}


start_bhyve()
{
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}For bhyve mode use: ${N2_COLOR}cbsd bstart jname=${jname} ${N1_COLOR}instead.${N0_COLOR}"
	exit 0
}

# chase 'grep -A28 ^20140930: /usr/ports/CHANGES'
set_changes_20140930()
{
	local _tpl="CBSD autotpl for CHANGES 20140930"

	[ ${applytpl} -eq 0 ] && return 0
	[ "${ver}" = "empty" ] && return 0

	set_bsdenv_by_path
	unset_bsdenv

	local nodever=$( ${miscdir}/elf_tables --freebsdver /bin/sh 2>/dev/null )
	local jailver=$( ${miscdir}/elf_tables --freebsdver ${path}/bin/sh 2>/dev/null )

	[ "${nodever}" = "${jailver}" ] && return 0

	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Applying /usr/ports/CHANGES -A28 ^20140930 work-around${N0_COLOR}"
	# we need put fake UNAME_r-stuff environment for correct works of port system

	set_bsdenv_by_path -p ${path} -v ${ver}

	# make backup and add records
	[ -f "${data}/etc/make.conf" ] && ${CP_CMD} ${data}/etc/make.conf ${data}/etc/make.conf.bak

	${CAT_CMD} >> ${data}/etc/make.conf <<EOF

OSVERSION+= ${OSVERSION}			# ${_tpl}
UNAME_ENV+= OSVERSION=${OSVERSION}		# ${_tpl}
UNAME_ENV+= UNAME_s=FreeBSD			# ${_tpl}
UNAME_ENV+= UNAME_r=${RELEASE}			# ${_tpl}
UNAME_ENV+= UNAME_v="\${UNAME_s} \${UNAME_r}"	# ${_tpl}
.MAKEFLAGS: \${UNAME_ENV}			# ${_tpl}
MAKE_ENV+= \${UNAME_ENV}			# ${_tpl}
CONFIGURE_ENV+= \${UNAME_ENV}			# ${_tpl}
SCRIPTS_ENV+= \${UNAME_ENV}			# ${_tpl}

EOF

	${CAT_CMD} >> ${data}/etc/profile << EOF

OSVERSION=${OSVERSION}; export OSVERSION		# ${_tpl}
UNAME_s=FreeBSD; export UNAME_s				# ${_tpl}
UNAME_r=${RELEASE}; export UNAME_v			# ${_tpl}
UNAME_v="\${UNAME_s} \${UNAME_r}"; export UNAME_v	# ${_tpl}

EOF

	${CAT_CMD} >> ${data}/etc/csh.cshrc << EOF

setenv OSVERSION ${OSVERSION}			# ${_tpl}
setenv UNAME_s FreeBSD				# ${_tpl}
setenv UNAME_r ${RELEASE}			# ${_tpl}
setenv UNAME_v "\${UNAME_s} \${UNAME_r}"	# ${_tpl}

EOF
	unset_bsdenv
}

unset_changes_20140930()
{
	local _tpl="CBSD autotpl for CHANGES 20140930"

	if ${GREP_CMD} "${_tpl}" ${data}/etc/make.conf >/dev/null 2>&1; then
		${CP_CMD} -a ${data}/etc/make.conf ${data}/etc/make.conf.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/make.conf.bak | ${GREP_CMD} "." > ${data}/etc/make.conf
	fi

	if ${GREP_CMD} "${_tpl}" ${data}/etc/profile >/dev/null 2>&1; then
		${CP_CMD} -a ${data}/etc/profile ${data}/etc/profile.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/profile.bak | ${GREP_CMD} "." > ${data}/etc/profile
	fi

	if ${GREP_CMD} "${_tpl}" ${data}/etc/csh.cshrc >/dev/null 2>&1; then
		${CP_CMD} -a ${data}/etc/csh.cshrc ${data}/etc/csh.cshrc.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/csh.cshrc.bak | ${GREP_CMD} "." > ${data}/etc/csh.cshrc
	fi
}

emulator="jail"	# for jname_is_multiple
jname_is_multiple

# MAIN for multiple jails
if [ $# -gt 1 -a -z "${jname}" -o -n "${jail_list}" ]; then

	# multiple astart always non interactive
	export inter=0
	# recursive
	if [ -n "${jail_list}" ]; then
		JLIST="${jail_list}"
	else
		JLIST=$*
	fi

	for jname in ${JLIST}; do
		[ "${jname}" = "inter=0" ] && continue
		${DAEMON_CMD} -p ${ftmpdir}/jstart.${jname}.$$ /usr/local/bin/cbsd jstart inter=0 jname=${jname} delay=1
		#lets save .pid file
		sleep 1
		[ -f "${ftmpdir}/jstart.${jname}.$$" ] && cbsd_pwait --pid=$( ${CAT_CMD} ${ftmpdir}/jstart.${jname}.$$ ) --timeout=${parallel}
	done

	wait_for_fpid -a start -t ${parallel}

	err 0 "${N1_COLOR}Multiple jstart: ${N2_COLOR}done${N0_COLOR}"
fi

# MAIN
. ${subrdir}/time.subr
[ -z "${jname}" ] && jname=${1}
. ${subrdir}/rcconf.subr

if [ $? -eq 1 ]; then
	# remote start
	[ ${sqlreplica} -eq 0 ] && err 1 "${N1_COLOR}No such jail: ${N2_COLOR}${jname}${N0_COLOR}"
	remotenode=$( jwhereis ${jname} )
	[ -z "${remotenode}" ] && err 1 "${N1_COLOR}No such jail: ${N2_COLOR}${jname}${N0_COLOR}"
	for i in ${remotenode}; do
		if [ "${i}" = "${nodename}" ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote jstart: found on nodename ${N2_COLOR}${nodename}${N1_COLOR}. Skipped${N0_COLOR}"
			continue
		fi
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remote jstart: ${N2_COLOR}${jname} ${N1_COLOR}on${N2_COLOR} ${i}${N0_COLOR}"
		rexe node=${i} cbsd jstart jname=${jname}
		if [ $? -eq 0 ]; then
			# updating state and put task for retrinv inventory
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Updating inventory...${N0_COLOR}"
			task autoflush=2 mode=new retrinv node=${i} tryoffline=1 data=db > /dev/null 2>&1
		fi
	done
	exit 0
fi

# mainly to smooth mass start in astart.
if [ ${delay} -ne 0 ]; then
	[ -z "${boot_delay}" ] && boot_delay=1
	cbsdlogger NOTICE ${CBSD_APP}: delayed boot for ${jname}: ${boot_delay} sec.
	sleep ${boot_delay}
fi

# start jail
st_time=$( ${DATE_CMD} +%s )

if [  "${ver}" = "native" ]; then
	tmpver=$( ${UNAME_CMD} -r )
	ver=${tmpver%%-*}
	unset tmpver
	[ "${stable}" = "1" ] && ver=${ver%%.*}
	cbsdsqlrw local "UPDATE jails SET ver=\"${ver}\" WHERE jname=\"${jname}\""
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}adjust version: ${N2_COLOR}native -> ${ver}${N0_COLOR}"
fi

if [ "${arch}" = "native" ]; then
	arch=$( ${UNAME_CMD} -m )
	target_arch="${arch}"
	cbsdsqlrw local "UPDATE jails SET arch=\"${arch}\" WHERE jname=\"${jname}\""
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}adjust version: ${N2_COLOR}native -> ${ver}${N0_COLOR}"
fi

over="${ver}"

# Determine stable value. Must be after buildconf
strpos --str="${over}" --search="."

# auto-detect for stable/release
pos=$?
if [ ${pos} -eq 0 ]; then
	stable=1
	ostable=1
else
	stable=0
	ostable=0
fi

[ ${status} -eq 2 ] && err 1 "${N1_COLOR}Jail in slave mode. Please ${N2_COLOR}cbsd jswmode mode=master${N1_COLOR} first${N0_COLOR}"
[ ${status} -eq 3 ] && err 1 "${N1_COLOR}Jail in maintenance mode${N0_COLOR}"
[ ${jid} -ne 0 ] && err 1 "${N1_COLOR}Jail ${jname} already running, jid: ${N2_COLOR}${jid}${N0_COLOR}"

if [ "${vnet}" = "1" -a "${vimage_feature}" = "0" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} have vnet=1 flags but your kernel is not support VIMAGE${N0_COLOR}"
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Please recompile kernel with: ${N2_COLOR}options VIMAGE${N0_COLOR}"
	sleep 3
	vnet=0
fi

[ "${emulator}" = "bhyve" -a -z "${mdsize}" ] && err 1 "${N1_COLOR}Bhyve required for file image${N0_COLOR}"

TRAP=""

#Check for shared lock
jaillock="${jailsysdir}/${jname}/locked"
if [ -f "${jaillock}" ]; then
	masterhost=$( ${CAT_CMD} ${jaillock} )
	if [ "${masterhost}" = "${nodename}" -o -z "${masterhost}" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: remove my stale lock file: ${jaillock}
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Remove my stale lock file: ${N2_COLOR}${jaillock}${N0_COLOR}"
		${RM_CMD} -f ${jaillock}
	else
		cur_time=$( ${DATE_CMD} +%s )
		eval $( ${STAT_CMD} -s ${jaillock} )
		difftime=$(( ( cur_time - st_mtime ) / 60 ))

		# 30 minutes outdated lock
		if [ ${difftime} -gt 30 ]; then
			cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file but lock file age is too old: ${difftime}. Removing!
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} locked by ${N2_COLOR}${masterhost}${N1_COLOR} node${N0_COLOR}"
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}But lock age is too old: ${difftime} min. Removing!!!${N0_COLOR}"
			${RM_CMD} -f ${jaillock}
		else
			# still fresh
			cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: locked by ${masterhost} node via ${jaillock} file and lock file age time is fresh: ${difftime}.
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Jail ${N2_COLOR}${jname}${N1_COLOR} locked by ${N2_COLOR}${masterhost}${N1_COLOR} node, lock age: ${difftime} min.${N0_COLOR}"
			log_err 1 "${N1_COLOR}You may remove the lockfile if you believe that jail is not running on this node: ${N2_COLOR}rm -f ${jaillock}${N0_COLOR}"
		fi
	fi
fi

TRAP="${TRAP} ${RM_CMD} -f ${jaillock};"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

#Check for md vnode backend
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	MDFILE="${jailsysdir}/${jname}/image.dat"
	if [ ! -f "${MDFILE}" -a ! -h "${MDFILE}" ]; then
		cbsdlogger NOTICE ${CBSD_APP}: jail: ${jname}: no such ${MDFILE} but mdsize flags is not null, skip
		[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}No such ${MDFILE} but mdsize flags is not null. Skip${N0_COLOR}" && continue
	fi
fi

# Update Redis
if [ "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
	cbsdredis hset "jail:${jname}" node "${nodename}" status 1 || echo "WARNING: failed to update Redis!"
	cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":1}'
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	if [ -z "${cbsd_queue_backend}" ]; then
		MOD_CBSD_QUEUE_DISABLED="1"
	else
		[ -n "${cbsd_jail_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_jail_queue_name} id=${jname} cmd=jstart status=1 workdir="${workdir}"
	fi
fi

[ "${emulator}" = "bhyve" ] && start_bhyve

init_target_arch
init_basedir

if [ ${baserw} -eq 0 ]; then
	if [ -r "${BASE_DIR_LOCKFILE}" ]; then
		locked_by=$( ${CAT_CMD} ${BASE_DIR_LOCKFILE} )
		log_err 1 "${N1_COLOR}basedir locked: ${N2_COLOR}${BASE_DIR}${N1_COLOR}, by pid: ${N2_COLOR}${locked_by}${N1_COLOR}. Please try later or remove ${BASE_DIR_LOCKFILE}${N0_COLOR}"
	fi
fi

jcleanup jname=${jname}

[ ! -d "${path}" ] && ${MKDIR_CMD} ${path}
[ -z "${mnt_start}" ] && mnt_start="0"

if [ "${mnt_start}" != "0" ]; then
	if [ ! -r "${mnt_start}" -o ! -x "${mnt_start}" ]; then
		err 1 "mnt_start script not exist or not executable: ${mnt_start}"
	fi
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Execute mnt_start script: ${N2_COLOR}${mnt_start}${N0_COLOR}..."
	# external mount, reset zfsfeat
	zfsfeat=0
	[ ! -d ${data} ] && ${MKDIR_CMD} -p ${data}
	[ ! -d ${jailfstabdir}/${jname} ] && ${MKDIR_CMD} -p ${jailfstabdir}/${jname}
	[ ! -d ${jailsysdir}/${jname} ] && ${MKDIR_CMD} -p ${jailsysdir}/${jname}
	${mnt_start} -d ${data} -f ${jailfstabdir}/${jname} -j ${jname} -r ${jailrcconfdir} -s ${jailsysdir}/${jname}
	_ret=$?
	if [ ${_ret} -ne 0 ]; then
		err 1 "${W1_COLOR}error: ${N1_COLOR}mnt_start script failed: ${N2_COLOR}${mnt_start} ${jname}${N0_COLOR}"
	fi
fi

check_environment_script "tests"

[ ! -r ${jailsysdir}/${jname}/local.sqlite ] && /usr/local/bin/cbsd ${miscdir}/updatesql ${jailsysdir}/${jname}/local.sqlite ${distdir}/share/local-jailnic.schema jailnic

# check for DHCP in ip4_addr string and replace them by ip
OIFS="${IFS}"
IFS=","
_tmp_ip4_addr=
_mod=0

for _tmp in ${ip4_addr}; do
	case ${_tmp} in
		[Dd][Hh][Cc][PP])
			_tmp=$( dhcpd )
			[ $? -eq 2 ] && log_err 1 "${N1_COLOR}No free IP address for DHCP in nodeippool${N0_COLOR}"
			cbsdlogger NOTICE ${CBSD_APP}: set ip4_addr for ${jname} via DHCP: ${_tmp}
			_mod=1
			;;
		[Dd][Hh][Cc][PP][vV]6)
			_tmp=$( dhcpdv6 )
			[ $? -eq 2 ] && log_err 1 "${N1_COLOR}No free IP6 address for DHCP in nodeip6pool${N0_COLOR}"
			cbsdlogger NOTICE ${CBSD_APP}: set ip4_addr for ${jname} via DHCPv6: ${_tmp}
			_mod=1
			;;
		*)
			;;
	esac

	if [ -z "${_tmp_ip4_addr}" ]; then
		_tmp_ip4_addr="${_tmp}"
	else
		_tmp_ip4_addr="${_tmp_ip4_addr},${_tmp}"
	fi
done

IFS="${OIFS}"

if [ ${_mod} -eq 1 ]; then
	jset jname=${jname} ip4_addr="${_tmp_ip4_addr}"
	ip4_addr="${_tmp_ip4_addr}"
fi

geniplist ${ip4_addr}

if [ -n "${interface}" -a "${interface}" != "0" -a "${vnet}" -eq "0" ]; then
	### CHECK FOR IP ####
	for ips in ${IPS}; do
		iptype "${ips}" ||true
		[ -z "${IWM}" ] && continue
		[ -n "${VHID}" ] && continue
		#prevent to use nodeip
		[ "${IWM}" = "${nodeip}" ] && err 1 "${N1_COLOR}Error: Jail can not take nodeip when interface is not equal 0: ${N2_COLOR}${nodeip}${N0_COLOR}"
		checkip ip=${IWM} check=1
		IPUSE=$?
		case ${IPUSE} in
			0)
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}ip ${IWM} not in pool range${N0_COLOR}"
				continue
			;;
			2)
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Ip ${IWM} already exists in LAN${N0_COLOR}"
				continue
			;;
		esac
	done
fi

# args for makejconf
epairb_list=
# for traffic count
epaira_list=

# extract and export/rewrite interface variable from jailnic for vnet-based jails
if [ "${vnet}" = "1" ]; then
	# for vnet we can make another action
	. ${subrdir}/vnet.subr

	interfaces=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT name FROM jailnic | while read _int; do
		printf "${_int} "
	done ) || err 1 "${N1_COLOR}jstart: error get interfaces name for vnet nic map${N0_COLOR}"

	eth_seq=0
	printf "${N1_COLOR}create epair: "

	for i in ${interfaces}; do

		nic_hwaddr=
		nic_parent=
		nic_address=
		_A=
		_A=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT nic_parent,nic_address,nic_hwaddr FROM jailnic WHERE name=\"${i}\" )
		sqllist "${_A}" nic_parent nic_address nic_hwaddr
		#nic_parent=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT nic_parent FROM jailnic WHERE name=\"${i}\" )

		# when ip=0 and interface=auto we must use default interface for upstream
		[ "${nic_parent}" = "0" -o "${nic_parent}" = "auto" ] && nic_parent=$( getnics-by-ip ip=0.0.0.0 skip=bridge )

		case "${nic_parent}" in
			vpc-*)
				VPC_ROOT_DIR="${dbdir}/vpc"
				_arg_len=$( strlen ${nic_parent} )
				vpc_name=$( substr --pos=5 --len=${_arg_len} --str="${nic_parent}" )
				_dbpath="${VPC_ROOT_DIR}/${vpc_name}.sqlite"
				[ ! -r ${_dbpath} ] && err 1 "${N1_COLOR}VPC not exist: ${N2_COLOR}${_dbpath}${N0_COLOR}"

				mybridge=
				if ! mybridge=$( get_my_device vpc vpc-${vpc_name} ); then
					err 1 "${N1_COLOR}${CBSD_APP} failed to get VPC bridge: ${mybridge}${N0_COLOR}"
				fi

				${IFCONFIG_CMD} ${mybridge} > /dev/null 2>&1
				_ret=$?
				if [ ${_ret} -ne 0 ]; then
					err 1 "${N1_COLOR}no such bridge interface: ${N2_COLOR}${mybridge}${N0_COLOR}"
				fi
				;;
			bridge*)
				mybridge="${nic_parent}"
				;;
			cbsdvale*)
				mybridge="${nic_parent}"
				;;
			*)
				if ! mybridge=$( get_my_device bridge ${nic_parent} ); then
					err 1 "${N1_COLOR}Error: Cant get_my_device for ${nic_parent}: ${mybridge}${N0_COLOR}"
				fi
		esac

		myepair=$( get_my_epair ${mybridge} )
		[ $? -eq 1 ] && err 1 "${N1_COLOR}Error: Cant get_my_epair by: ${N2_COLOR}${mybridge}${N0_COLOR}"
		if [ -z "${epairb_list}" ]; then
			epaira_list="${myepair}a"
			epairb_list="${myepair}b"
		else
			epaira_list="${epaira_list},${myepair}a"
			epairb_list="${epairb_list},${myepair}b"
		fi

		case "${nic_parent}" in
			cbsdvale*)
				strpos --str="${nic_parent}" --search="_"
				_pos=$?
				if [ ${_pos} -eq 0 ]; then
					# not vale_XXX form
					err 1 "${N1_COLOR}${CBSD_APP}: vale switch not vale_XXX form: ${N2_COLOR}${nic_parent}${N0_COLOR}"
				else
					_arg_len=$( strlen ${nic_parent} )
					_pref=$(( _arg_len - _pos ))
					_vale_arg=$( substr --pos=0 --len=${_pos} --str="${nic_parent}" )
					_sw=$( substr --pos=$(( ${_pos} +2 )) --len=${_pref} --str="${nic_parent}" )
				fi
				valeid=$( cbsdsqlro local SELECT idx FROM vale WHERE name=\"${_sw}\"  )
				[ -z "${valeid}" ] && log_err 1 "${N1_COLOR}${CBSD_APP}: can't determine vale nic for: ${nic_parent}, sw: ${_sw}${N0_COLOR}"
				#mytap="vale${valeid}:${jname}"
				cbsdlogger NOTICE "${CBSD_APP}: vale switch id: ${valeid} (vale${valeid})"
				[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CBSD_APP}: attach to vale switch id: ${valeid} (${N2_COLOR}vale${valeid}${N1_COLOR})${N0_COLOR}"
				${distdir}/tools/vale-ctl -h vale${valeid}:${myepair}a
		esac

		printf "${H3_COLOR}${myepair}:${N2_COLOR}${nic_parent} "
		${IFCONFIG_CMD} ${myepair}a description ${jname}-eth${eth_seq} up
		eth_seq=$(( eth_seq + 1 ))
		TRAP="${TRAP} ${IFCONFIG_CMD} ${myepair}a destroy;"
		trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

		#nic_hwaddr=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT nic_hwaddr FROM jailnic WHERE name=\"${i}\" )

		# MAC MGMT
		if [ "${nic_hwaddr}" = "0" ]; then
			nic_hwaddr=$( mac_gen 00:a0:98 )
			cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite "UPDATE jailnic SET nic_hwaddr=\"${nic_hwaddr}\" WHERE name=\"${i}\""
		fi

		cbsdlogger NOTICE ${CBSD_APP}: jail ${jname}: set hwaddr for vnet interface ${myepair}b: ${nic_hwaddr}
		${IFCONFIG_CMD} ${myepair}b ether ${nic_hwaddr}
		if [ "${nic_address}" != "0" ]; then
			${IFCONFIG_CMD} ${myepair}a inet ${nic_address} up
		fi
	done

	[ ${quiet} -ne 1 ] && ${ECHO} "${N0_COLOR}"
fi	# vnet -eq 1

#test for zfs mounted & mount if not
case ${zfsfeat} in
	1)
		. ${subrdir}/zfs.subr
		zfsmnt ${data}
		if [ $? -eq 2 ]; then
			# ZFS encryption?
			zfs_encryption_val=$( ${ZFS_CMD} get -Ho value encryption ${ZPOOL} 2>/dev/null )
			if [ -n "${zfs_encryption_val}" -a "${zfs_encryption_val}" != "off" ]; then
				${ECHO} "${N1_COLOR}${CBSD_APP}: encrypted dataset: ${N2_COLOR}${zfs_encryption_val}${N0_COLOR}"
				zfs_encryption_keyformat=$( ${ZFS_CMD} get -Ho value keyformat ${ZPOOL} 2>/dev/null )
				${ECHO} "${N1_COLOR}${CBSD_APP}: encryption keyformat: ${N2_COLOR}${zfs_encryption_keyformat}${N0_COLOR}"
				if [ "${zfs_encryption_keyformat}" = "passphrase" ]; then
					if [ -n "${NOINTER}" -o "${inter}" = "1" ]; then
						err 0 "${N1_COLOR}${CBSD_APP} keyformat is passphrase, skipp for non-interactive mode${N0_COLOR}"
					else
						${ZFS_CMD} load-key ${ZPOOL}
						# 255 code valid here, already loaded
					fi
				fi
			fi
			${ZFS_CMD} mount "${ZPOOL}"
			is_mounted ${data} || err 1 "${N1_COLOR}${CBSD_APP}: unable to mount zfs pool: ${N2_COLOR}${ZPOOL} -> ${data}${N0_COLOR}"
		fi
	;;
esac

if [ "${ver}" != "empty" ]; then
	if [ ${baserw} -eq 1 ]; then
		path=${data}
		[ ! -f "${path}/bin/sh" ] && switch_baserw ${path} 1
		[ -f ${mount_fstab} ] && ${RM_CMD} -f ${mount_fstab}
	fi

	if [ ${baserw} -eq 0 -a ! -f "${mount_fstab}" ]; then
		switch_baserw ${path} 2
	fi
fi

# MD area
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	cbsd mountmd mdfile="${MDFILE}" jroot="${data}"
fi

if [ "${ver}" != "empty" ]; then
	[  -d "${data}/etc" -o -d "${data}/bin" ] || err 1 "${N1_COLOR}No such data structure: ${N2_COLOR}${data}${N0_COLOR}"
fi

# cp local default resolv.conf skel
if [ ${floatresolv} -eq 1 -a "${ver}" != "empty" ]; then
	makeresolv jname=${jname}
fi

if [ ${applytpl} -eq 1 -a "${ver}" != "empty" ]; then
	${SYSRC_CMD} -qf ${data}/etc/rc.conf hostname="${host_hostname}" >/dev/null
fi

MOUNTOPT=""

if [ "${mount_src}" = "1" ]; then
	SRCDIR="${srcdir}/src_${ver}/src"
	MOUNTOPT="${MOUNTOPT} -s ${SRCDIR}"
fi

if [ "${mount_obj}" = "1" ]; then
	SRCDIR="${srcdir}/obj_${arch}_${ver}/obj"
	MOUNTOPT="${MOUNTOPT} -o ${SRCDIR}"
fi

if [ "${mount_kernel}" = "1" ]; then
	if [ -d ${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel ]; then
		SRCDIR="${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel"
		MOUNTOPT="${MOUNTOPT} -k ${SRCDIR}"
	fi
fi

if [ "${mount_ports}" = "1" ]; then
	case "${platform}" in
		DragonFly)
			MOUNTOPT="${MOUNTOPT} -p /usr/dports"
			;;
		*)
			MOUNTOPT="${MOUNTOPT} -p /usr/ports -d ${data}/var/cache/distfiles"
			;;
	esac
fi

[ "${ver}" != "empty" ] && mountbase -v ${ver} -a ${arch} ${MOUNTOPT}

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
with_zfs_attach="${jailsysdir}/${jname}/with_zfs_attach"
[ -r ${with_zfs_attach} ] && ${RM_CMD} -f ${with_zfs_attach}

mount_jail_fstab
export_jail_data_for_external_hook
external_exec_master_script "master_prestart.d"

if [ "${ver}" = "empty" ]; then
	#path="/"
	[ -z "${exec_start}" ] && exec_start="${jailsysdir}/${jname}/run.sh"
else
	[ ! -d "${data}/var/cache/pkg" ] && ${MKDIR_CMD} -p "${data}/var/cache/pkg"
fi

#determine that jail is FreeBSD. Useful for vnet operation in makejconf and
is_freebsd=0

if [ ${baserw} -eq 1 ]; then
	elftest=${data}/bin/sh
else
	elftest="${BASE_DIR}/bin/sh"
fi
[ -f "${elftest}" ] && osname=$( ${miscdir}/elf_tables --osname ${elftest} )

[ "${osname}" = "freebsd" ] && is_freebsd=1

# check for freshed version of the base via elf from /bin/sh
if [ ${is_freebsd} -eq 1 -a ${baserw} -eq 1 ]; then
	if [ -n ${BASE_DIR} -a -f "${BASE_DIR}/bin/sh" ]; then
		baseelf=$( ${miscdir}/elf_tables --ver ${BASE_DIR}/bin/sh 2>/dev/null )
		jailelf=$( ${miscdir}/elf_tables --ver ${elftest} 2>/dev/null )
		if [ ${baseelf} -gt ${jailelf} ]; then
			[ ${quiet} -ne 1 ] && ${ECHO} "${BOLD}Notice: ${N1_COLOR}You have a more recent version of the base in ${BASE_DIR} (${N2_COLOR}${baseelf}${N1_COLOR}/${N2_COLOR}${jailelf}${N1_COLOR}).${N0_COLOR}"
			[ ${quiet} -ne 1 ] && ${ECHO} "${BOLD}Notice: ${N1_COLOR}Please consider upgrading jail base via ${N2_COLOR}cbsd jupgrade${N0_COLOR}"
		fi
	fi
fi

set -e
makejconf jname=${jname} out=${ftmpdir}/${jname}.conf ip6wa=${HAVE_IPV6} epair=${epairb_list} fbsd=${is_freebsd} quiet=${quiet}
set +e

#rctl/limits area
jrctl jname=${jname} mode=set quiet=${quiet}
[ -r "${jailsysdir}/${jname}/helpers/jrctl.sqlite" ] && nice=$( cbsdsqlro ${jailsysdir}/${jname}/helpers/jrctl.sqlite "SELECT cur FROM forms WHERE param=\"nice\"" )
[ -z "${nice}" ] && nice="0"
[ "${nice}" != "0" ] && [ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}jail renice: ${N2_COLOR}${nice}${N0_COLOR}"

unset_changes_20140930
set_changes_20140930

etcupdate_check

[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}Starting jail: ${N2_COLOR}${jname}, parallel timeout=${parallel}${N0_COLOR}"
TRAP="${TRAP} ${RM_CMD} -f ${ftmpdir}/jstart.${jname}.$$;"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

# check that the emulator is able to execute commands
res=$( emulator_exec_check )

ret=$?

if [ ${ret} -ne 0 ]; then
	[ ${quiet} -ne 1 ] && $ECHO "${N1_COLOR}emulator error: ${N2_COLOR}${res}${N0_COLOR}"
	jcleanup jname="${jname}"
	exit 0
fi

# TODO/NOTES:
# when path have no /var and/or /tmp (e.g empty fstab file), after jail start we have
# undesirable/undesirable tmpfs! bug/issue ?
# tmpfs on /usr/jails/jails/jail2/var (tmpfs, local)
# tmpfs on /usr/jails/jails/jail2/tmp (tmpfs, local)

case "${platform}" in
	DragonFly)
		# legacy jail tools only
		#cat ${ftmpdir}/${jname}.conf
		${MOUNT_CMD} -t devfs devfs ${path}/dev
		echo "[debug] ${JAIL_CMD} ${path} ${host_hostname} ${ip4_addr} ${exec_start}"
		${DAEMON_CMD} -p ${ftmpdir}/jstart.${jname}.$$ /usr/bin/nice -n ${nice} ${JAIL_CMD} ${path} ${host_hostname} ${ip4_addr} ${exec_start}
		echo "[debug]: ${ftmpdir}/jstart.${jname}.$$"
		;;
	*)
		case "${quiet}" in
		0)
			${DAEMON_CMD} -p ${ftmpdir}/jstart.${jname}.$$ /usr/bin/nice -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname}
			;;
		1)
			${DAEMON_CMD} -p ${ftmpdir}/jstart.${jname}.$$ /usr/bin/nice -n ${nice} ${JAIL_CMD} -f ${ftmpdir}/${jname}.conf -c ${jname} > /dev/null
			;;
		esac
esac

trap "" HUP INT ABRT BUS TERM EXIT

[ -f "${ftmpdir}/jstart.${jname}.$$" ] && cbsd_pwait --pid=$( ${CAT_CMD} ${ftmpdir}/jstart.${jname}.$$ ) --timeout=${parallel}

### late start ###
# late exec_start: for zfsattached and vnet-based jails
# this is necessary for any operations that must be performed after
# creating the jail but before running real exec_start
late_start=

# sign of zfs attach inside jail: we need special route for this case
# remove orphaned sign if exist: then touched it by mountfstab script
if [ -r ${with_zfs_attach} -a ${is_freebsd} -eq 1 ]; then
	[ -r ${mount_fstab} ] && attachzfs fstab=${mount_fstab} jname=${jname}
	[ -r ${mount_fstab}.local ] && attachzfs fstab=${mount_fstab}.local jname=${jname}
	# real exec_start for zfs attached jail
	${JEXEC_CMD} ${jname} ${ZFS_CMD} mount -a
	late_start="${exec_start}"
fi

# for VNET-based jail we also need special route
# due to jail.conf doesn't support for multiple NICs
if [ ${vnet} -eq 1 ]; then
	OIFS="${IFS}"
	IFS=","
	eth_seq=0
	for i in ${epairb_list}; do
		[ -z "${i}" ] && continue
		IFS="${OIFS}"
		${IFCONFIG_CMD} ${i} up
		# we need setup lock here due to ethX name possible conflict with other jail
		while [ -r ${tmpdir}/eth${eth_seq}.locked ]; do
			[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}jstart: eth${eth_seq} locked, waiting..${N0_COLOR}"
			sleep 1
		done
		# make lock
		echo "${jname}" > ${tmpdir}/eth${eth_seq}.locked
		${IFCONFIG_CMD} ${i} name eth${eth_seq} && ${IFCONFIG_CMD} eth${eth_seq} vnet ${jname}
		case "${ip4_addr}" in
			[Rr][Ee][Aa][Ll][Dd][Hh][Cc][Pp])
				# this file can be managed by users and/or 3rd party stuff (e.g. Reggae)
				[ -r ${workdir}/jails-data/${jname}-data/etc/rc.conf.d/network ] && . "${workdir}/jails-data/${jname}-data/etc/rc.conf.d/network"
				if [ -z "${dhclient_program}" ]; then
					DHCLIENT_PROGRAM=`${JEXEC_CMD} ${jname} sysrc -n dhclient_program`
				else
					DHCLIENT_PROGRAM=${dhclient_program}
				fi
				${JEXEC_CMD} ${jname} ${DHCLIENT_PROGRAM} eth${eth_seq}
				;;
			0|[Dd][Hh][Cc][Pp])
				# just skip
				;;
			*)
				# alias ? IPv6 ? mtu ?
				# we need to re-build ifconfig_eth0 vars if values already exist to merge new IP address and
				# and do not spoil the remaining data

				# Whats about Linux jail?
				if [ ! -r ${data}/etc/rc.conf ]; then
					${ECHO} "${W1_COLOR}warning: ${N1_COLOR}no such /etc/rc.conf: ${N2_COLOR}${name}${N1_COLOR}, skip for config ifconfig_eth0${N0_COLOR}"
				else
					ifconfig_eth0=
					new_ifconfig_eth0="${ip4_addr}"
					eval $( ${GREP_CMD} ^ifconfig_eth0= ${data}/etc/rc.conf 2>/dev/null | ${HEAD_CMD} -n1 )
					#echo "ifconfig old val [${ifconfig_eth0}]"
					if [ -n "${ifconfig_eth0}" ]; then
						${ECHO} "${N1_COLOR}adjust IPv4 address for eth0: ${N2_COLOR}${ip4_addr}${N0_COLOR}" 1>&2
						for _part in ${ifconfig_eth0}; do
							iptype ${_part}
							if [ $? -eq 1 ]; then
								#echo "!! IPv4 detected: ${_part}, skipp"
								continue
							fi
							if [ -z "${new_ifconfig_eth0}" ]; then
								new_ifconfig_eth0="${_part}"
							else
								new_ifconfig_eth0="${new_ifconfig_eth0} ${_part}"
							fi
						done
					fi
					#echo "NEW VAL: [${new_ifconfig_eth0}]"
					${SYSRC_CMD} -qf ${data}/etc/rc.conf ifconfig_eth0="${new_ifconfig_eth0}" > /dev/null 2>&1
				fi
		esac

		# release lock
		${RM_CMD} -f ${tmpdir}/eth${eth_seq}.locked
		eth_seq=$(( eth_seq + 1 ))
		IFS=","
	done

	if [ -n "${ci_gw4}" ]; then
		case ${ci_gw4} in
			0)
				# do nothing
				;;
			auto)
				# always first?
				nic_address=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT nic_address FROM jailnic LIMIT 1 )
				# is ip?
				if [ -n "${nic_address}" -a "${nic_address}" != "0" ]; then
					echo "${N1_COLOR}${CBSD_APP}: set defaultrouter to nic_address: ${N2_COLOR}[${nic_address}]${N0_COLOR}"
					${SYSRC_CMD} -qf ${data}/etc/rc.conf defaultrouter="${nic_address}" > /dev/null 2>&1
				elif [ -n "${oci_gw4}" ];then
					nic_address="${oci_gw4}"
					echo "${N1_COLOR}${CBSD_APP}: set defaultrouter to by config file: ${N2_COLOR}[${nic_address}]${N0_COLOR}"
					${SYSRC_CMD} -qf ${data}/etc/rc.conf defaultrouter="${nic_address}" > /dev/null 2>&1
				else
					echo "${N1_COLOR}${CBSD_APP}: ci_gw4 sets to auto bit ic_address is empty${N0_COLOR}"
				fi
				;;
			*)
				${SYSRC_CMD} -qf ${data}/etc/rc.conf defaultrouter="${ci_gw4}" > /dev/null 2>&1
				;;
		esac
	fi

	IFS="${OIFS}"
	late_start="${exec_start}"
fi

if [ -n "${late_start}" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}late_start in progress...${N2_COLOR}"
	${JEXEC_CMD} ${jname} ${late_start}
fi
### /late start ###

# update state_time
cbsdsqlrw local UPDATE jails SET state_time="(strftime('%s','now'))" WHERE jname=\"${jname}\"

# make id file
UNDHOSTNAME=$( echo ${hostname} | ${TR_CMD}  "." "_" )
FID="/var/run/jail_${UNDHOSTNAME}.id"
echo ${ST} > ${FID}
exec_cbsdjail_first_boot

external_exec_script -s start.d

get_jid
jid=${myjid}

if [ "${platform}" = "DragonFly" ]; then
	[ ${allow_raw_sockets} -eq 1 ] && ${SYSCTL_CMD} -w jail.${jid}.net_raw_sockets=1
fi

external_exec_master_script "master_poststart.d"

if [ ${myjid} -gt 0 ]; then
	status="1"
	# make shared lock
	echo "${nodename}" > ${jaillock}
	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${jaillock}
else
	status="0"
	# looks like jail start is failed, execute post script
	exec_master_poststop
	external_exec_master_script "master_poststop.d"
	exec_poststop
	jcleanup jname=${jname}

	# redis
	if [ "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
		cbasredis hset "jail:${jname}" status -1 || echo "WARNING: Failed to update Redis"
		cbasredis hdel "jail:${jname}" jid || echo "WARNING: Failed to update Redis"
		cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":-1}'
	fi
fi

cbsdsqlrw local "UPDATE jails SET jid=${myjid},status=${status} where jname='${jname}'"
${RM_CMD} -f ${ftmpdir}/${jname}.conf
fwcounter
expose mode=apply

if [ ${myjid} -gt 0 ]; then
	if [ -r "${jailsysdir}/${jname}/cpu" ]; then
		. ${jailsysdir}/${jname}/cpu
		echo "${CPUSET_CMD} -c -l ${cpuset} -j ${myjid}" 1>&2
		${CPUSET_CMD} -c -l ${cpuset} -j ${myjid}
	else
		if [ ${cpuset} != "0" ]; then
			echo "${CPUSET_CMD} -c -l ${cpuset} -j ${myjid}" 1>&2
			${CPUSET_CMD} -c -l ${cpuset} -j ${myjid}
		fi
	fi
fi

# VNC auto start
if [ -x "${distmoduledir}/vncterm.d/cbsdvnc" ]; then
	[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}vncterm is installed, launch vncterm session...${N0_COLOR}"
	${DAEMON_CMD} -f /usr/local/bin/cbsd vncterm jname=${jname} mode=run addr=127.0.0.1 >/dev/null 2>&1
fi

# CBSD QUEUE
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	[ -n "${cbsd_jail_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_jail_queue_name} id=${jname} cmd=jstart status=2 data_status=1 workdir="${workdir}"
fi

end_time=$( ${DATE_CMD} +%s )
diff_time=$(( end_time - st_time ))
# redis
if [ ${myjid} -gt 0 -a "${mod_cbsd_redis_enabled}" = "YES" -a -z "${MOD_CBSD_REDIS_DISABLED}" ]; then
	cbsdredis hset "jail:${jname}" status 2 jid ${myjid} started ${st_time} echo "WARNING: Failed to update Redis"
	cbsdredis publish cbsd_events '{"cmd":"jstart", "node":"'${nodename}'", "jail":"'${jname}'", "status":0, "duration":'${diff_time}'}'
fi
diff_time=$( displaytime ${diff_time} )

if [ -f "${jailsysdir}/${jname}/jail-message" -a ${quiet} -ne 1 ]; then
	echo "---- jail messages ----" 1>&2
	${SED_CMD} -e s:\$ip4_addr:${ip4_addr}:g "${jailsysdir}/${jname}/jail-message" 1>&2
	echo "---- jail messages ----" 1>&2
fi

[ ${quiet} -ne 1 ] && ${ECHO} "${N1_COLOR}${CBSD_APP} done ${N2_COLOR}in ${diff_time}${N0_COLOR}"
cbsdlogger NOTICE ${CBSD_APP}: jail ${jname} started in ${diff_time}

exit 0
