#
# Copyright 2017 SAS Institute Inc.
# SAS Campus Drive, Cary, North Carolina 27513, USA.
# All rights reserved.
#
#
# Version 2.0.00
#
# This script automates the management of UNIX/Linux based multi-tiered services.
# Please refer to SASNote http://support.sas.com/kb/58/231.html for details and updates
# 
# REVISION HISTORY
# ----------------
#
# Date        Developer   	Change
# ----        ---------   	------
# 2016-07-11  Clifford Meyers	birthday
# 2017-06-01  Clifford Meyers	add log capture functionality
#

function usage 
{
	echo ""
	echo "<<USAGE>> [bash | sh] ${PROG} [-a <NUM> | -o <NUM> | -s] -c <CFG> [-e]"
	echo ""
	echo "  WHERE"
	echo "    -a NUM     start deployment tier services from tier NUM to MAXTIERS"
	echo "    -o NUM     stop deployment tier services from MAXTIERS to tier NUM"
	echo "    -s         provide status of all deployment tier services"
	echo "    -c CFG     specify configuration file"
	echo "    -e         optionally extract potential tier error logs to a centralized location under !STATUSROOT"
	echo ""
	echo "  NOTE"
	echo "    -a, -o, and -s options are not to be run concurrently" 
	echo ""
	exit 1
}

function initialize 
{
	CRC=0
	UNAME=`uname -a | awk '{ print $1 }'`
	case "${ACTION}" in
	  start|stop|status)	;;
	  *)			usage
	  			;;
	esac
	if [ ! -f ${DIR}/${CFGFILE} ]
	  then	echo "<<ERROR>> missing configuration file [${CFGFILE}] from [${DIR}]"
		usage
	  else	. ${DIR}/${CFGFILE}
		STATUSROOT=${STATUSROOT:-/tmp}
		if [ ! -w "${STATUSROOT}" ]
		  then	echo "<<ERROR>> Status report root directory [${STATUSROOT}] is not writable"
			exit 1
		  else	STATUSDIR=${STATUSROOT}/${CFGFILE}
			mkdir -p ${STATUSDIR} >/dev/null 2>&1
			if [ $? -ne 0 ]
		  	  then	echo "<<ERROR>> Cannot create STATUSDIR directory [${STATUSDIR}]"
				exit 1
			fi
			OUTFILE=${STATUSDIR}/${PROG}.outfile.${RANDOM}
			rm -f ${STATUSDIR}/${PROG}.* >/dev/null 2>&1
			REPORT=${STATUSDIR}/report.${ACTION}
        		IS_RUNNING=`ps -fu $(whoami) | grep ${PROG} | egrep -vc grep`
        		if [ "${IS_RUNNING}" -gt 2 ]
          	  	  then  echo "<<ERROR>> script [${PROG}] is currently being executed" 
				exit 1
			  else	LANG=C
				DRC=0
				TRC=0
		  	  	echo "COMMAND: ${COMMAND}" >${REPORT}
		  		validate_cfgfile
			fi
		fi
	fi
}

function validate_cfgfile 
{
	echo "" | tee -a ${REPORT}
	echo "Configuration file validation [${CFGFILE}] status [begin]" | tee -a ${REPORT}
	TRIMOFFSET=${TRIMOFFSET:-1}
	TEST=`expr ${TRIMOFFSET} - 0 2>/dev/null`
	if [ -z "${TEST}" ]
	  then	echo "  <<ERROR>> TRIMOFFSET value [${TRIMOFFSET}] is not valid.  Valid TRIMOFFSET values are positive whole numbers." | tee -a ${REPORT}
		CRC=1
	fi
        TEST=`expr ${SLEEP} - 0 2>/dev/null`
	if [ -z "${TEST}" ]
	  then	echo "  <<ERROR>> MAXRETRY value [${MAXRETRY}] is not valid.  Valid MAXRETRY values are positive whole numbers." | tee -a ${REPORT}
		CRC=1
	fi
        TEST=`expr ${SLEEP} - 0 2>/dev/null`
        if [ -z "${TEST}" ]
	  then	echo "  <<ERROR>> SLEEP value [${SLEEP}] is not valid.  Valid SLEEP values are positive whole numbers." | tee -a ${REPORT}
		CRC=1
	fi
        TEST=`expr ${MAXTIERS} - 0 2>/dev/null`
        if [ -z "${TEST}" ]
	  then	echo "  <<ERROR>> MAXTIERS value [${MAXTIERS}] is not numeric." | tee -a ${REPORT}
		CRC=1
	  else	((INCR=1))
	  	while [ ${INCR} -le ${MAXTIERS} ]
	  	  do
 			if [ -z "${TIERNAME[$INCR]}" ]
		  	  then	TIERDRCCHK="TIERNAME"
			fi
  			if [ -z "${TIERINST[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERINST"
			  else	INST=${TIERINST[$INCR]}
			fi
  			if [ -z "${TIERHOST[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERHOST"
		  	  else	HOST=${TIERHOST[$INCR]}
			fi
  			if [ -z "${TIERSTAR[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERSTAR"
			fi
  			if [ -z "${TIERSTOP[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERSTOP"
			fi
  			if [ -z "${TIERREST[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERREST"
			fi
  			if [ -z "${TIERSTAT[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERSTAT"
			fi
  			if [ -z "${TIERSTAS[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERSTAS"
			fi
  			if [ -z "${TIERSTOS[$INCR]}" ]
		  	  then	TIERDRCCHK="${TIERDRCCHK} TIERSTOS"
			fi
			testssh nomsg
			if [ $DRC -ne 0 ]
			  then	echo "  <<ERROR>> TIER[${INCR}]: ${CMD}" | tee -a ${REPORT}
				CRC=1
			fi
			if [ ! -z "${TIERDRCCHK}" ]
			  then	CRC=1
				echo "  <<ERROR>> Validation of TIER[${INCR}] failed, missing or invalid items:"  | tee -a ${REPORT}
				for ITEM in ${TIERDRCCHK}
				  do
			  		echo "          ${ITEM}"  | tee -a ${REPORT}
				  done
				TIERDRCCHK=""
			fi
			((INCR=INCR+1))
	  	  done
	fi
	if [ ${CRC} -eq 1 ]
	  then	UNKNOWN=true
		echo "Configuration file validation [${CFGFILE}] status [end failed - 000]" | tee -a ${REPORT}
		echo "" | tee -a ${REPORT}
		cleanup
	  else	echo "Configuration file validation [${CFGFILE}] status [end success]" | tee -a ${REPORT}
		echo "" | tee -a ${REPORT}
	fi
}

function get_dom
{
   case ${M} in
     2)         if [ `expr ${Y} % 400` -eq 0 ]
                  then  D=29
                  else  if [ `expr ${Y} % 100` -eq 0 ]
                          then  D=28
                          else  if [ `expr ${Y} % 4` -eq 0 ]
                                  then  D=29
                                  else  D=28
                                fi
                        fi
                fi
                ;;
   4|6|9|11)    D=30
                ;;
   *)           D=31
                ;;
   esac
}

function bttf
{
	DATE=`date +%Y-%m-%d-%H`
	Y=`echo ${DATE} | awk -F\- '{ print $1 }'`
	M=`echo ${DATE} | awk -F\- '{ print $2 }' | sed "s;^0;;g"`
	D=`echo ${DATE} | awk -F\- '{ print $3 }' | sed "s;^0;;g"`
	H=`echo ${DATE} | awk -F\- '{ print $4 }' | sed "s;^0;;g"`
	case "${ACTION}" in
	  status)	((H=H-$TRIMOFFSET))
			;;
	  start|stop)	;;
	esac
	if [ ${H} -lt 0 ]
  	  then  ((D=D-1))
		((H=H+24))
        	if [ ${D} -lt 1 ]
          	  then  ((M=M-1))
                	get_dom
                	if [ ${M} -lt 1 ]
                  	  then  ((Y=Y-1))
                        	M=12
                	fi
		fi
        fi
	M=`printf %.2d ${M}`
	D=`printf %.2d ${D}`
	H=`printf %.2d ${H}`
	HA="${Y}-${M}-${D} ${H}"
}

function create_BOM
{
	CT=${1}
	if [ ! -f ${STATUSDIR}/logfiles.BOM ]
	  then	bttf
	  	echo "${HA}" > ${STATUSDIR}/logfiles.BOM
		echo "" >>${STATUSDIR}/logfiles.BOM
	fi
	TMPFILE1=/tmp/tmpfile1.${RANDOM}
	TMPFILE2=/tmp/tmpfile2.${RANDOM}
	echo "${CONF}" > ${TMPFILE1}
	for RECORD in `ssh ${INST}@${HOST} "find ${CONF} -type f -name \"*.srv\" -print"`
 	  do
		ssh ${INST}@${HOST} "cat ${RECORD}" | egrep "^/" >> ${TMPFILE1}
 	  done
	for RECORD in `cat ${TMPFILE1} | awk '{ print length, $0 }' | sort -nu | awk '{ print $2 }' | egrep -v ${CONF}/`
  	  do
		ssh ${INST}@${HOST} "find ${RECORD} -mmin -60 -type f -name \"*.log\" -print" >>${TMPFILE2}
  	  done
	if [ -s ${TMPFILE2} ]
  	  then	echo "Server [${HOST}] [${NAME}] issues" >> ${STATUSDIR}/logfiles.BOM
		echo "" >> ${STATUSDIR}/logfiles.BOM
		echo "<<TIER SERVICES>>" 	 >> ${STATUSDIR}/logfiles.BOM
		ssh ${INST}@${HOST} "${STAT} | egrep \"${CT}\"" >> ${STATUSDIR}/logfiles.BOM
		echo "" >> ${STATUSDIR}/logfiles.BOM
		echo "<<  LOG FILES  >>" >> ${STATUSDIR}/logfiles.BOM
		cat ${TMPFILE2} |
		while read RECORD
		  do
			echo "${INST}:${HOST}:${TIER}:${RECORD}" >> ${STATUSDIR}/logfiles.BOM
		  done 
		echo "" >> ${STATUSDIR}/logfiles.BOM
	fi
	rm -f ${TMPFILE1} ${TMPFILE2} >/dev/null 2>&1
}

function trim_logs
{
	TOKEN=`head -1 ${STATUSDIR}/logfiles.BOM`
	YEAR=`echo ${TOKEN} | awk -F\- '{ print $1 }'`
	MONT=`echo ${TOKEN} | awk -F\- '{ print $2 }'`
	MON=`date +%b`
	DAY=`echo ${TOKEN} | awk -F\- '{ print $3 }' | awk '{ print $1 }'`
	HOUR=`echo ${TOKEN} | awk '{ print $2 }'`
	TOKEN2=`echo ${TOKEN} | sed "s; ;T;"`
	TOKEN3=`echo ${TOKEN} | sed "s;-;/;g"`
	TOKEN4="${DAY}-${MONT}-${YEAR} ${HOUR}"
	TOKEN5="${YEAR}${MONT}${DAY}:${HOUR}"
	TOKEN6="${DAY} ${MON} ${YEAR} ${HOUR}"
	TOKEN7="${DAY}/${MON}/${YEAR}:${HOUR}"
	TOKEN8="${YEAR}-${MONT}-${DAY} ${HOUR}"
	echo "`date`: action [trim logfiles] status [begin]" | tee -a ${REPORT}
	find ${STATUSDIR}/logfiles -type f |
	while read FILE
  	  do
		FOUND=`egrep -n "${TOKEN}|${TOKEN2}|${TOKEN3}|${TOKEN4}|${TOKEN5}|${TOKEN6}|${TOKEN7}|${TOKEN8}" ${FILE} | head -1 | awk -F\: '{ print $1 }'`
		if [ ! -z "${FOUND}" ]
	  	  then	FS=`wc -l ${FILE} | awk '{ print $1 }'`
			((TAIL=FS-FOUND+1))
			tail -${TAIL} ${FILE} >${FILE}.trimmed
			mv ${FILE}.trimmed ${FILE}
		fi
  	  done
	echo "`date`: action [trim logfiles] status [end success]" | tee -a ${REPORT}
	echo "" | tee -a ${REPORT}
}

function extract_logs
{
	LOGROOT=${STATUSDIR}/logfiles
	TMPFILE1=/tmp/tmpfile1.${RANDOM}
	echo "`date`: action [centralize logfiles] status [begin]" | tee -a ${REPORT}
	grep "log\$" ${STATUSDIR}/logfiles.BOM | awk -F\: '{ print $2"/"$3"/"$4 }' | awk 'BEGIN{FS=OFS="/"}{$NF=""; NF--; print}' | sort -u | xargs -i mkdir -p ${LOGROOT}/{} 
	grep "log\$" ${STATUSDIR}/logfiles.BOM | sort -u >${TMPFILE1}
	for LOGRECORD in `cat ${TMPFILE1}`
  	  do
		INST=`echo ${LOGRECORD} | awk -F\: '{ print $1 }'`	
		HOST=`echo ${LOGRECORD} | awk -F\: '{ print $2 }'`	
		TIER=`echo ${LOGRECORD} | awk -F\: '{ print $3 }'`	
		LOGF=`echo ${LOGRECORD} | awk -F\: '{ print $4 }'` 
		case "${UNAME}" in
		HP-UX)	AVAIL=`df -k ${LOGROOT} | grep free | awk '{ print $1 }'`
			;;
		AIX)	AVAIL=`df -k ${LOGROOT} | tail -1 | awk '{ print $3 }'` 
			;;
		*)	AVAIL=`df -k ${LOGROOT} | tail -1 | awk '{ print $4 }'` 
			;;
		esac
		SIZE=`ssh ${INST}@${HOST} du -s -k ${LOGF} | awk '{ print $1 }'`
		if [ "${AVAIL}" -gt "${SIZE}" ]
	  	  then	scp -p ${INST}@${HOST}:${LOGF} ${LOGROOT}/${HOST}/${TIER}/${LOGF} >/dev/null 2>&1
			TRIM=true
	  	  else	echo "  <<  WARNING  >> insufficient space to copy log to [${LOGROOT}]"
			TRIM=false
			rm -rf ${LOGROOT} >/dev/null 2>&1
			break
		fi
  	  done 
	echo "`date`: action [centralize logfiles] status [end]" | tee -a ${REPORT}
	echo "" | tee -a ${REPORT}
	if [ "${TRIM}" = "true" ]
	  then	trim_logs
	fi
	rm -f ${TMPFILE1} >/dev/null 2>&1
}

function print_message 
{
	((PMINCR=$1))
	TITLE="`date`: action [${2}] tier [${PMINCR}:${NAME}] host [${HOST}] status [$3]"
	echo "${TITLE}" | tee -a ${REPORT}
}

function run_cmd 
{
	if [ "${VERBOSE}" != "nomsg" ]
	  then	echo "  << EXECUTING >> ${CMD}" | tee -a ${REPORT}
	fi
	eval ${CMD} >${OUTFILE} 2>&1
	DRC=$?
	if [ ${DRC} -ne 0 ] && [ "${VERBOSE}" != "nomsg" ]
	  then	cat ${OUTFILE} | egrep -v "Stopping" |
		while read ERRREC
		  do
			echo "                  ${ERRREC}" | tee -a ${REPORT}
		  done
	fi
}

function testssh 
{
	VERBOSE=$1
	CMD="ssh ${INST}@${HOST} -o 'BatchMode=yes' -o 'ConnectionAttempts=1' true"
	run_cmd 
	VERBOSE=""
}

function get_array 
{
	GAINCR=${1}
	NAME=${TIERNAME[$GAINCR]}
	INST=${TIERINST[$GAINCR]}
	HOST=${TIERHOST[$GAINCR]}
	STAR=${TIERSTAR[$GAINCR]}
	STOP=${TIERSTOP[$GAINCR]}
	REST=${TIERREST[$GAINCR]}
	STAT=${TIERSTAT[$GAINCR]}
        case ${STAT} in
          /*)	CONF=${STAT%/*}
                ;;
	  *)	CONF=""
		;;
        esac
}

function check_services 
{
	((TRY=1))
	((CHECK=1))
	CMD="ssh ${INST}@${HOST} ${STAT}"  
	while [ ${TRY} -le ${MAXRETRY} ] && [ ${CHECK} -ne 0 ]
	  do
		run_cmd
		if [ $DRC -ne 0 ]
		  then	echo "  <<TIER STATUS>> unable to determine tier status" |  tee -a ${REPORT}
			UNKNOWN=true
			echo "                  `cat ${OUTFILE}`"
			return
		fi
		CHECK=`cat ${OUTFILE} | egrep -c "${CHECKTOKEN}"`
		if [ ${CHECK} -ne 0 ]
		  then	((TRY=TRY+1))
		  	sleep ${SLEEP}
		  else 	DRC=0
		fi
	  done
	SRK=`egrep -c "${CHECKTOKEN}|${GOODTOKEN}" ${OUTFILE}`
	if [ ${SRK} -eq 0 ]
	  then	echo "  <<TIER STATUS>> unable to determine tier status" |  tee -a ${REPORT}
	  	UNKNOWN=true
		echo "                  `cat ${OUTFILE}`"
	  else
		if [ ${CHECK} -ne 0 ]
	  	  then	echo "  <<TIER STATUS>> ALL tier services have NOT returned the proper status [${GOODTOKEN}]" | tee -a ${REPORT}
                	TRC=1
                	cat ${OUTFILE} | grep "${CHECKTOKEN}" |
                	while read TOKENRECORD
                  	  do
                		echo "                  ${TOKENRECORD}" | tee -a ${REPORT}
                  	  done
			if [ ! -z "${CONF}" ]
			  then  create_BOM "${CHECKTOKEN}"  
			fi
	  	  else	echo "  <<TIER STATUS>> ALL tier services have returned the proper status [${GOODTOKEN}]" | tee -a ${REPORT}
		fi
	fi
}

function check_dependencies 
{
	CDINCR=${TIER}
        TEST=`expr ${CDINCR} - 0 2>/dev/null`
	if [[ -z "${TEST}" ]] || [ ${CDINCR} -gt ${MAXTIERS} ] || [ ${CDINCR} -lt 1 ]
	  then	echo "<<ERROR>> \"start from\" tier value [${TIER}] is not valid.  Valid tier values are [1-${MAXTIERS}]" | tee -a ${REPORT}
		CDINCR=1
                while [ ${CDINCR} -le ${MAXTIERS} ]
                  do
                        echo "          ${CDINCR}   ${TIERNAME[$CDINCR]}" | tee -a ${REPORT}
                        ((CDINCR=CDINCR+1))
                  done
		echo "" | tee -a ${REPORT}
		DRC=1
		UNKNOWN=true
	  else  ((CDINCR=1))
		while [ "${CDINCR}" -le "${MAXTIERS}" ] && [ ${DRC} -eq 0 ] && [ "${UNKNOWN}" != "true" ]
	  	  do
			get_array ${CDINCR}
			if [ "${CDINCR}" -lt "${TIER}" ]
		  	  then	EXPECTED=started
				CHECKTOKEN=${TIERSTAS[$CDINCR]}
				GOODTOKEN=${TIERSTOS[$CDINCR]}
		  	  else	EXPECTED=stopped
				CHECKTOKEN=${TIERSTOS[$CDINCR]}
				GOODTOKEN=${TIERSTAS[$CDINCR]}
			fi
			if [ "${CDINCR}" -ne "${TIER}" ]
			  then	print_message "${CDINCR}" "check dependencies tier ${EXPECTED}" "begin"
				testssh 
				if [ ${DRC} -eq 0 ]
		  	  	  then	check_services 
					if [ "${CHECK}" -eq 0 ]
			    	  	  then  print_message "${CDINCR}" "check dependencies tier ${EXPECTED}" "end success"
			  	  	  else  print_message "${CDINCR}" "check dependencies tier ${EXPECTED}" "end failed - 001"
						DRC=2
						TRC=2
					fi
		  	  	  else  print_message "${CDINCR}" "check_dependencies" "end failed - 002"
					DRC=1
					TRC=2
				fi
				echo "" | tee -a ${REPORT}
			fi
			((CDINCR=CDINCR+1))
	  	  done
	fi
}

function start_services 
{
	rm -rf ${STATUSDIR}/logfiles.BOM ${STATUSDIR}/logfiles ${STATUSDIR}/.lastaction >/dev/null 2>&1 
	check_dependencies
	while [ ${DRC} -eq 0 ] && [ ${TIER} -le ${MAXTIERS} ] && [ "${UNKNOWN}" != "true" ]
	  do
		get_array ${TIER}
		CHECKTOKEN=${TIERSTAS[$TIER]}
		GOODTOKEN=${TIERSTOS[$TIER]}
		print_message "${TIER}" "${ACTION}" "begin"
		testssh
		if [ ${DRC} -eq 0 ]
		  then	if [ ${TIER} -ne ${TIERSTART} ] 
			  then	CMD="ssh ${INST}@${HOST} ${STAR}" 
			  else	CMD="ssh ${INST}@${HOST} ${REST}"
			fi
			run_cmd
			check_services 
			if [ "${CHECK}" -eq 0 ]
			  then	print_message "${TIER}" "${ACTION}" "end success"
			  else	print_message "${TIER}" "${ACTION}" "end failed - 003"
				DRC=1
			fi
		  else	print_message "${TIER}" "${ACTION}" "end failed - 004"
			DRC=1
		fi
		echo "" | tee -a ${REPORT}
		((TIER=TIER+1))
	  done	
}
 
 
function stop_services 
{
	rm -rf ${STATUSDIR}/logfiles.BOM ${STATUSDIR}/logfiles ${STATUSDIR}/.lastaction >/dev/null 2>&1
	((TIER=MAXTIERS))
        TEST=`expr ${TIERSTOP} - 0 2>/dev/null`
	if [[ -z "${TEST}" ]] || [ ${TIERSTOP} -gt ${MAXTIERS} ] || [ ${TIERSTOP} -lt 1 ]
	  then	echo "<<ERROR>> \"stop to\" tier value [${TIERSTOP}] is not valid.  Valid tier values are [${MAXTIERS}-1]" | tee -a ${REPORT}
		CDINCR=${MAXTIERS}
                while [ ${CDINCR} -ge 1 ]
                  do
                        echo "          ${CDINCR}   ${TIERNAME[$CDINCR]}" | tee -a ${REPORT}
                        ((CDINCR=CDINCR-1))
                  done
		echo "" | tee -a ${REPORT}
		DRC=1
		UNKNOWN=true
	  else	while [ ${TIER} -ge ${TIERSTOP} ] && [ ${DRC} -eq 0 ] && [ "${UNKNOWN}" != "true" ]
	  	  do
			get_array ${TIER}
			CHECKTOKEN=${TIERSTOS[$TIER]}
			GOODTOKEN=${TIERSTAS[$TIER]}
			print_message "${TIER}" "${ACTION}" "begin"
			testssh
			if [ ${DRC} -eq 0 ]
		  	  then	CMD="ssh ${INST}@${HOST} ${STOP}"
				run_cmd
				check_services 
				if [ "${CHECK}" -eq 0 ]
			  	  then	print_message "${TIER}" "${ACTION}" "end success"
			  	  else	print_message "${TIER}" "${ACTION}" "end failed - 005"
					DRC=1
				fi
		  	  else	print_message "${TIER}" "${ACTION}" "end failed - 006"
				DRC=1
			fi
			echo "" | tee -a ${REPORT}
			((TIER=TIER-1))
	  	  done	
	fi
}

function cleanup 
{
	LASTACTION=`cat ${STATUSDIR}/.lastaction 2>/dev/null | awk -F\: '{ print $1 }'`
	LASTACTION=${LASTACTION:-$ACTION}
	if [ "${UNKNOWN}" = "true" ]
  	  then 	echo "<<DEPLOYMENT STATUS>> Deployment tier services are in an unknown state"  | tee -a ${REPORT}
		STATUS=FAILURE
		TRC=1
		DRC=2
	  else	case ${DRC} in
		  0)	STATUS=SUCCESS
			;;
		  1|2) 	STATUS=FAILURE
			;;
		esac
  	  	case "${ACTION}" in
		  start)	case ${TRC} in
				  0)  	MSG="<<${STATUS}>> Deployment tier services are operational" 
					;;
				  1)  	MSG="<<${STATUS}>> Deployment tier services are NOT fully operational"
					;;
				  2)  	MSG="<<${STATUS}>> Deployment tier services dependency check failed" 
					rm -f ${STATUSDIR}/logfiles.BOM >/dev/null 2>&1
					DRC=0
					;;
				esac
				;;
		  stop) 	STAN=${TIERNAME[$MAXTIERS]}
				STON=${TIERNAME[$TIERSTOP]}
				case ${TRC} in
				  0)  	STATE="NOT"
					;;
				  1)    STATE="NOT fully"
					;;
				esac
				MSG="<<${STATUS}>> Deployment tier services are ${STATE} operational from [${MAXTIERS}:${STAN}] to tier [${TIERSTOP}:${STON}]" 
				;;
		  status)	DOWN=`grep -c "ALL tier services are NOT operational" ${REPORT}`
				UP=`grep -c "ALL tier services are operational" ${REPORT}`
				XX=`grep -c "ALL tier services are NOT fully operational" ${REPORT}`
				if [ ${DOWN} -gt 0 -a ${UP} -gt 0 ] || [ ${XX} -gt 0 ]
				  then	MSG="<<STATUS>> Deployment tier services are NOT fully operational" 
					TRC=1
				  else	if [ ${DOWN} -eq 0 ]
					  then	MSG="<<STATUS>> Deployment tier services are operational"  
						if [ ${LASTACTION} = "start" ]
						  then	rm -rf ${STATUSDIR}/logfiles.BOM ${STATUSDIR}/logfiles >/dev/null 2>&1
						fi
					  else	MSG="<<STATUS>> Deployment tier services are NOT operational"
						if [ ${LASTACTION} = "stop" ]
						  then	rm -rf ${STATUSDIR}/logfiles.BOM ${STATUSDIR}/logfiles >/dev/null 2>&1
						fi
					fi
				fi
				;;
		esac
	fi
	if [ -f "${STATUSDIR}/logfiles.BOM" ] && [ `wc -l ${STATUSDIR}/logfiles.BOM | awk '{ print $1 }'` -gt 2  ]
	  then	if [ "${EXTRACTLOGS}" = "true" ] && [ ! -d ${STATUSDIR}/logfiles ] 
	 	  then	mkdir ${STATUSDIR}/logfiles >/dev/null 2>&1
			cp -p ${CFGFILE} ${STATUSDIR}/logfiles/
			extract_logs
		fi
	  	echo "Please see ${REPORT} for execution details" | tee -a ${REPORT}
		echo "           ${STATUSDIR}/logfiles.BOM for a listing of logfiles" | tee -a ${REPORT}
		if [ -d ${STATUSDIR}/logfiles ]
		 then	echo "           ${STATUSDIR}/logfiles for the centralized logfiles" | tee -a ${REPORT}
		fi
	  	echo "${LASTACTION}:1" > ${STATUSDIR}/.lastaction
	  else	rm -f ${STATUSDIR}/logfiles.BOM 2>/dev/null
		echo "Please see ${REPORT} for execution details" | tee -a ${REPORT}
		if [ "${ACTION}" != "status" ]
	  	  then	echo "${ACTION}:${DRC}" > ${STATUSDIR}/.lastaction
	  	  else	echo "${LASTACTION}:${DRC}" > ${STATUSDIR}/.lastaction
		fi
	fi
	echo ""
	echo "${MSG}" | tee -a ${REPORT}
	echo ""
	if [ ! -z "${SASADMIN}" -a "${ACTION}" != "status" ]
	  then	mailx -s "`date` `hostname` ${PROG} ${STATUS} ${ACTION}" ${SASADMIN} < ${REPORT} 
	fi
	rm -f ${OUTFILE} >/dev/null 2>&1
	exit ${DRC}
}

function tier_status 
{
        ((TRY=1))
        ((DRC=1))
	UPTOKEN=${TIERSTAS[$TIER]}
	DOWNTOKEN=${TIERSTOS[$TIER]}
        CMD="ssh ${INST}@${HOST} ${STAT}"
        run_cmd
        while [ ${TRY} -le ${MAXRETRY} ] && [ ${DRC} -ne 0 ]
          do
                run_cmd
		((TRY=TRY+1))
          done
	if [ ${DRC} -eq 0 ]
	  then	UPCOUNT=`egrep -c "${DOWNTOKEN}" ${OUTFILE}`
		DOWNCOUNT=`egrep -c "${UPTOKEN}" ${OUTFILE}`
		if [ ${UPCOUNT} -gt 0 -a ${DOWNCOUNT} -eq 0 ]
	  	  then	echo "  <<TIER STATUS>> ALL tier services are operational" | tee -a ${REPORT}
		  else	if [ ${UPCOUNT} -eq 0 -a ${DOWNCOUNT} -gt 0 ]
		  	  then	echo "  <<TIER STATUS>> ALL tier services are NOT operational" | tee -a ${REPORT}
		  	  else	echo "  <<TIER STATUS>> ALL tier services are NOT fully operational" | tee -a ${REPORT}
			fi
                	CHECK=`cat ${STATUSDIR}/.lastaction 2>/dev/null` 
			if [ "${CHECK}" = "start:0" -a  ! -z "${CONF}" ]
			  then	create_BOM "${UPTOKEN}" 
			fi
		fi
		cat ${OUTFILE} | egrep -v "SAS servers status:" |
		while read TOKENRECORD
		  do
			if [ -n "${TOKENRECORD}" ]
			  then	echo "                  ${TOKENRECORD}" | tee -a ${REPORT}
			fi
		  done
	fi
}

function status_services 
{
        ((TIER=1))
        while [ ${TIER} -le ${MAXTIERS} ] 
          do
                get_array ${TIER}
                print_message "${TIER}" "services_status" "begin"
		testssh
		if [ ${DRC} -eq 0 ]
		  then	tier_status
			if [ ${DRC} -eq 0 ]
			  then	print_message "${TIER}" "services_status" "end success"
	  	  	  else	echo "  <<TIER STATUS>> unable to determine tier status" |  tee -a ${REPORT}
				UNKNOWN=true
				print_message "${TIER}" "services_status" "end failed - 007"
			fi
		fi
		echo "" | tee -a ${REPORT}
		((TIER=TIER+1))
	  done
}


# main
CURRSHELL=`ps -p $$ | tail -1 | awk '{print $4}'`
if [ "${CURRSHELL}" != "bash" -a "${CURRSHELL}" != "sh" ] 
  then	echo "<<ERROR>> Invalid invocation of script via shell [${CURRSHELL}]. Please invoke via the bash or bourne (sh) shell"
	exit 1
fi
PROG=${0##*/}
DIR=${0%/*}
if [ "${DIR}" = "${PROG}" ]
  then	DIR=.
fi
COMMAND="${PROG} ${*}"
args=`getopt esc:a:o: $*`
for i in $args
  do
        case "$i" in
          -c)   CFGFILE=$2
                shift 2
                ;;
          -a)   ACTION="${ACTION}start"
                TIER=$2
		TIERSTART=${TIER}
	        shift 2
                ;;
          -o)   ACTION="${ACTION}stop"
                TIER=$2
		TIERSTOP=${TIER}
	        shift 2
                ;;
          -s)   ACTION="${ACTION}status"
                shift
		;;
          -e)   EXTRACTLOGS=true
                shift
		;;
        esac
  done
initialize
case "${ACTION}" in
  start)	start_services 
		;;
  stop)		stop_services 
		;;
  status)	status_services
		;;
esac
cleanup
