#!/usr/bin/env bash

set -e

if [ -z "$IFO" ] ; then
    HOST=$GUARD_HOST
else
    if [[ "$IFO" == "L1" ]] ; then
	HOST=${GUARD_HOST:="l1script0"}
    else
	HOST=${GUARD_HOST:="${IFO,,*}guardian0"}
    fi
fi
USER=${GUARD_USER:=controls}

CHANNELS=(
    'VERSION'
    'MODE'
    'ERROR'
    'NOTIFICATION'
    'STATUS'
    'NOMINAL_N'
    'REQUEST_N'
    'STATE_N'
    'TARGET_N'
    'OK'
)

##################################################

PGRM=$(basename $0)

_usage() {
    cat <<EOF
usage: $PGRM <command> [<system> ...]

Guardian daemon control.

The following commands are for creating/controlling guardian daemon
processes running under guardian-runsvdir on host:

  ${USER}@${HOST:=???}

See 'guardian --usage' for more info.

commands:
  create <nodes>                create nodes
                                <node> should match the name of an
                                existing guardian system module in
                                USERAPPS.  A node for the specified
                                system will be created with the same
                                name. The created node will not be
                                started.  If '--all', all existing
                                nodes will be recreated.

  destroy <nodes>               destroy nodes
  start <nodes>                 start nodes
  stop <nodes>                  stop nodes
  restart <nodes>               restart nodes
  list [<nodes>]                display nodes (all if none specified)

  mode MODE <nodes>             set MODE for nodes
  request STATE <nodes>         set REQUEST for nodes

  log [<option>] <nodes>        tail node logs.  Multiple nodes may be
                                specified, in which case their logs will
                                be interleaved.
    -c                            cat log, instead of tail
    -l <nlogs>                    with -c, cat last <nlogs> log files
                                  (default: 10)

"<nodes>" indicates either a single node, a space-separated list of
nodes, or "--all" to indicate all nodes.
EOF
  # update [<path> [...]]     update guardian USERAPPS checkout
  #                           <path> should be relative to USERAPPS root
}

_log() {
    echo "$@" 1>&2
}

##################################################

_ssh() {
    local pgrm="$1"
    local cmd="$2"
    shift 2
    mkdir -p ~/.ssh/controls/
    # setup a master connection and use ControlPersist to hold it
    # open for a while for fast subsequent calls
    local ctrl_path="~/.ssh/controls/%r@%h:%p"
    local sshcmd="ssh -q -S ${ctrl_path} \
                      -o ControlMaster=auto \
                      -o ControlPersist=600 \
                      -t ${USER}@${HOST}"
    $sshcmd "$pgrm" "$cmd" \"${@}\"
}

##################################################

_setenv() {
    for var in 'IFO' 'SITE' ; do
	if [ -z "${!var}" ] ; then
	    _log "'$var' environment variable not defined."
	    exit 1
	fi
    done

    local site="${SITE,,*}"
    local ifo="${IFO,,*}"

    TARGET=${GUARD_TARGET:=/etc/guardian}
    NODEDIR="$TARGET"/nodes
    export SVDIR="$TARGET"/service
    #LOGDIR=/var/log/guardian
    LOGDIR="$TARGET"/logs
    #SUPERVISEDIR=/run/guardian
    SUPERVISEDIR="$TARGET"/supervise
    CHANFILE=${GUARD_CHANFILE:=/opt/rtcds/"$site"/"$ifo"/chans/daq/"$IFO"EDCU_GRD.ini}

}

_nodelist() {
    if [ -z "$1" ] ; then
	:
    elif [[ "$1" == '--all' ]] ; then
	(cd "$NODEDIR" && ls -1)
    else
	if ! (cd "$NODEDIR" && ls -1d $@) 2>/dev/null ; then
	    echo "$@"
	fi
    fi
}

_nodechan() {
    local node="$1"
    local chan="$2"
    echo "${IFO}:GRD-${1}_${2}"
}

_node_get() {
    local node="$1"
    local chan="$2"
    shift 2
    caget $@ $(_nodechan "$node" "$chan")
}

_node_put() {
    local node="$1"
    local chan="$2"
    local value="$3"
    shift 3
    caput $@ $(_nodechan "$node" "$chan") "$value"
}

_get_node_stat() {
    local nodedir="$NODEDIR"/"$1"
    cat "$nodedir"/supervise/stat
}

_is_active() {
    local node="$1"
    local svdir="$SVDIR"/"$node"
    [ -e "$svdir" ]
}

_is_running() {
    [[ $(_get_node_stat "$node") != 'down' ]]
}

_list() {
    local format="$1"
    local node="$2"

    local nodedir="$NODEDIR"/"$node"
    local svdir="$SVDIR"/"$node"

    local super='-'
    local ok='-'
    local mode='-'
    local version='-'
    local state='-'
    local msg='-'

    # node dir present
    if [ ! -e "$nodedir" ] ; then
	_log "unknown node: $node"
	return
    fi

    # node is linked in
    if [ -e "$svdir" ] ; then
	# check if service link is ok
	if [ -L "$svdir" ] && [[ $(readlink "$svdir") == "$nodedir" ]] ; then
	    if [ -L "$svdir"/supervise ] ; then
		case $(_get_node_stat "$node") in
		    'down')
			super='o'
			#msg="use 'start' command to start node"
			;;
		    *)
			super='*'
                        IFS=$'\n' data=($(caget -t -S ${IFO}:GRD-${node}_{OK,MODE,ERROR,USERMSG,STATE_S,VERSION}))
                        ok=${data[0]}
			if [[ "$ok" == 'True' ]] ; then
			    ok='*'
			else
			    ok=' '
			fi
                        mode=${data[1]::1}
                        error=${data[2]}
                        msg=$(echo ${data[3]} | sed 's/ *$//')
                        if [[ 'error' == 'True' ]] ; then
                            mode='!'
                            msg='NODE ERROR!'
                        fi
                        state=${data[4]}
                        version=${data[5]}
			;;
		esac
		#msg=$(sv status "$node" || true)
	    else
		# if a newly created rundir is linked in and hasn't
		# been detected by runsvdir yet, it will exist but the
		# supervise directory will not have been created at
		# the sv status will fail
		# FIXME: maybe more robust check against the output of
		# sv status is in order
		super='.'
		msg='init...'
	    fi
	else
	    super='!'
	    msg='service directory improperly linked'
	fi
    else
	super='-'
	msg='-'
    fi

    printf "$format" \
	"$node" \
	"$super" \
	"$ok" \
	"$mode" \
	"$version" \
	"$state" \
	"$msg"
}

##################################################

_create() {
    local node="$1"
    local nodedir="${NODEDIR}/${node}"
    local logdir="${LOGDIR}/${node}"
    local supervisedir="${SUPERVISEDIR}/${node}"

    _log "creating node $node..."

    mkdir -p "$nodedir"
    mkdir -p "$logdir"
    mkdir -p "$nodedir"/log
    mkdir -p "$nodedir"/env

    # create 'guardian' file with node name
    echo "$node" >"$nodedir"/guardian

    # create the run script
    cat <<EOF >"$nodedir"/run
#!/bin/bash
set -e
exec 2>&1
if [ -e $TARGET/local-env ] ; then
  . $TARGET/local-env
fi
exec chpst -e env guardian \$(cat guardian) "\$@"
EOF
    chmod 755 "$nodedir"/run

    # create the finish script
    cat <<'EOF' >"$nodedir"/finish
#!/bin/bash
echo guardian process stopped: $*
# FIXME: do this in a more flexible way
node=$(cat guardian)
echo "guardian $node died: $*" | mail -s "guardian-runsv: $node died: $*" jameson.rollins@ligo.org
EOF
    chmod 755 "$nodedir"/finish

    # create the log run script
    cat <<'EOF' >"$nodedir"/log/run
#!/bin/bash
set -e
LOG=$(readlink -f ./main)
test -d "$LOG" || mkdir -p -m0750 "$LOG"
exec svlogd "$LOG"
EOF
    chmod 755 "$nodedir"/log/run

    # create a log config
    cat <<'EOF' >"$nodedir"/log/config
# don't delete old log files
n0
EOF
    # create the log dir
    mkdir -p "$logdir"
    # link log main to the logdir
    ln -sfn "$logdir" "$nodedir"/log/main
    # link in the log config file
    ln -sfn "$nodedir"/log/config "$logdir"/config

    # create the service down
    touch "$nodedir"/down

    # make symlinks for the supervise directories
    ln -sfn "$supervisedir" "$nodedir"/supervise
    ln -sfn "$supervisedir".log "$nodedir"/log/supervise
}

_destroy() {
    local node="$1"
    local nodedir="${NODEDIR}/${node}"
    local supervisedir="${SUPERVISEDIR}/${node}"
    local logdir="${LOGDIR}/${node}"

    _log "destroying node $node..."

    rm -f "$nodedir"/guardian
    rm -f "$nodedir"/run
    rm -f "$nodedir"/supervise
    rm -f "$nodedir"/finish
    rm -f "$nodedir"/down
    rm -f "$nodedir"/log/config
    rm -f "$nodedir"/log/run
    rm -f "$nodedir"/log/main
    rm -f "$nodedir"/log/supervise
    rm -f "$nodedir"/env/*
    [ -d "$nodedir"/env ] && rmdir "$nodedir"/env
    [ -d "$nodedir"/log ] && rmdir "$nodedir"/log
    [ -d "$nodedir" ] && rmdir "$nodedir"

    rm -f "$supervisedir"{,.log}/control
    rm -f "$supervisedir"{,.log}/lock
    rm -f "$supervisedir"{,.log}/ok
    rm -f "$supervisedir"{,.log}/pid
    rm -f "$supervisedir"{,.log}/stat
    rm -f "$supervisedir"{,.log}/status
    [ -d "$supervisedir" ] && rmdir "$supervisedir"
    [ -d "$supervisedir".log ] && rmdir "$supervisedir".log

    rm -f "$logdir"/lock
}

_add() {
    local node="$1"
    _log "adding node $node..."
    ln -sfn "$NODEDIR"/"$node" "$SVDIR"/"$node"
}

_remove() {
    local node="$1"
    _log "removing node $node..."
    rm -f "$SVDIR"/"$node"
}

_start() {
    local node="$1"
    _log "starting node $node..."
    # we do a 'once' here because we don't want it to restart if there
    # was a fatal error
    sv once "$node"
    # starting the process implies that we also want it started at
    # boot
    rm -f "$NODEDIR"/"$node"/down
}

_stop() {
    local node="$1"
    _log "stopping node $node..."
    [ -e "$NODEDIR"/"$node"/supervise/ok ] && sv stop "$node"
    # stopping the process implies that we also don't want it started
    # at boot
    touch "$NODEDIR"/"$node"/down
}

##################################################

_tree() {
    # FIXME: better way to find runsvdir pid
    local pid=$(initctl status guardian-runsvdir | awk '{print $4}')
    if [ -z "$pid" ] ; then
	_log "could not find running guardian-runsvdir process."
	exit 1
    fi
    pstree -acplnG $pid
}

##################################################

_viewlog() {
    local temp
    local output='follow'
    local nlogs=10
    local nodes
    local logdir

    temp=$(getopt -o cl: --long cat,limit: -n "$PGRM log" -- "$@")
    if [ $? -ne 0 ] ; then
        _log "Invalid options."
	exit 1
    fi
    # note the quotes around `$temp': they are essential!
    eval set -- "$temp"
    while true ; do
        case "$1" in
            -c|--cat)
		output='cat'
		shift
		;;
	    -l|--limit)
		nlogs="$2"
		shift 2
		;;
            --)
		shift
		;;
            *)
		nodes=($(_nodelist "$@"))
		break
		;;
        esac
    done

    case $output in
	'follow')
	    logdirs=(${nodes[@]/#/"${LOGDIR}/"})
	    logs=(${logdirs[@]/%/'/current'})
	    exec tail -q -n 1000 -F "${logs[@]}"
	    ;;
	'cat')
	    if ((${#nodes[@]} > 1)) ; then
		_log "Currently only single node log cat is supported."
		exit 1
	    fi
	    logdir="${LOGDIR}/${nodes}"
	    for log in $(ls -tr1 "$logdir"/\@* | tail -"$nlogs") ; do
		cat "$log"
	    done
	    cat "$logdir"/current
	    ;;
    esac
}

##################################################

_update_global_chanfile() {
    _log "updating global chans file: $CHANFILE ..."
    cat <<EOF >"$CHANFILE".new
[default]
gain=1.00
datatype=4
ifoid=0
slope=6.1028e-05
acquire=3
offset=0
units=V
dcuid=4
datarate=16
EOF
    for node in $(_nodelist --all) ; do
	for channel in ${CHANNELS[@]} ; do
	    printf '[%s:GRD-%s_%s]\n' "$IFO" "$node" "$channel" >>"$CHANFILE".new
	done
    done
    mv -f "$CHANFILE"{.new,}
}

_update() {
    cd "$TARGET"/userapps/release
    _log updating: "${@:-ALL}"
    svn update "$@"
}

##################################################
##################################################

if (($# < 1)) ; then
    _usage
    exit 1
fi

if [ "$HOST" ] && [[ $(hostname) != "$HOST" ]] && [[ "$HOST" != 'localhost' ]] ; then
    _ssh true || (_log "Could not connect to host '${HOST}'."; exit 10)
    _ssh $PGRM ${@}
    exit $?
fi

cmd="${1,,}"
shift

case "$cmd" in
    nodes)
	_setenv
	_nodelist "$@"
	;;
    list)
	_setenv
	format='%-12s %s %s %s %4s  %-40s %s\n'
	if [[ "$1" != '-h' ]] ; then
	    printf "$format" \
		node s k m vers state message
	    printf "$format" \
		---- - - - ---- ----- -------
	else
	    shift
	fi
	if [ -z "$1" ] ; then
	    nodes='--all'
	else
	    nodes="$@"
	fi
	for node in $(_nodelist "$nodes") ; do
	    _list "$format" "$node"
	done
	;;
    create)
	_setenv
	for node in $(_nodelist "$@") ; do
	    if [[ "$node" =~ /+ ]] ; then
		_log "Systems may not be specified by path."
		exit 1
	    fi
	    guardian --print "$node" >/dev/null
	    _create "$node"
	    _add "$node"
	    _log "guardian node created:"
	    cd "$NODEDIR"/"$node"
	    ./run --print
	done
	_update_global_chanfile
	# FIXME: wait for nodes to be recognized by runsvdir
	;;
    destroy)
	_setenv
	for node in $(_nodelist "$@") ; do
	    if _is_active "$node" ; then
		_stop "$node"
	    fi
	    _remove "$node"
	    _destroy "$node"
	done
	_update_global_chanfile
	;;
    start)
	_setenv
	for node in $(_nodelist "$@") ; do
	    _start "$node"
	done
	;;
    stop)
	_setenv
	for node in $(_nodelist "$@") ; do
	    _stop "$node"
	done
	;;
    restart)
	_setenv
	for node in $(_nodelist "$@") ; do
	    if [[ "$1" == '--all' ]] && ! _is_running "$node" ; then
		continue
	    fi
	    _stop "$node"
	    _start "$node"
	done
	;;
    mode|request)
	_setenv
	value="$1"
	shift
	for node in $(_nodelist "$@") ; do
	    _node_put "$node" "${cmd^^}" "$value"
	done
	;;
    reload)
	_setenv
	for node in $(_nodelist "$@") ; do
	    _node_put "$node" MODE RELOAD
	done
	;;
    tree)
	_tree
	;;
    log)
	_setenv
	_viewlog $@
	;;
    print)
	_setenv
	cd "$NODEDIR"/"$1"
	./run --print
	;;
    update)
	_setenv
	_update "$@"
	;;
    env)
	_setenv
	echo TARGET=$TARGET
	echo SVDIR=$SVDIR
	echo NODEDIR=$NODEDIR
	echo LOGDIR=$LOGDIR
	echo SUPERVISEDIR=$SUPERVISEDIR
	;;
    help|-h|--help)
	_usage
	_log
	_setenv
	;;
    sv)
	_setenv
	sv "$@"
	;;
    *)
	_log "Unknown command: $cmd"
	_log "See '$PGRM help' for more info."
	exit 1
	;;
esac