#!/bin/bash
#
# fetch-data.sh v1.2 2006/04
#
# Shell script for remote multiple command execution on multiple servers
# for monitoring purposes.
#
# Copyright (C) 2006 by Pedro Venda < pjvenda (at) pjvenda org >
#
# Distributed under the terms of the GNU Public License Agreement (GPLv2)
# http://www.fsf.org/licensing/licenses/gpl.html or
# http://www.fsf.org/licensing/licenses/gpl.txt
# 

#
# Usage
# =====
#
# Setup configuration files:
#   * cmd.conf (cmd:cmd_output_file:cmd_description)
#   * hosts.conf (host1
# 		  host2)
#   * comments are prefixed with '#' and ignored by the script
#
# Run script
#   ./fetch-data.sh
#   (best used with ssh keys, of course)
#
# Changelog
# =========
#
# v1.2 - 2006/04/02
# - dependency checks (external binaries)
# - cleanups, file and directory checks
# - optional logging to stdout
# - better new_dir routine
#
# v1.1 - 2006/03/29
# - cleanups and light optimizations
#
# v1.0 - 2006/03/21 
# - initial version
#

# relevant binaries
SSH=/usr/bin/ssh
RM=/bin/rm
DATE=/bin/date
CUT=/bin/cut
EGREP=/bin/egrep
CAT=/bin/cat
MKDIR=/usr/bin/mkdir

# directories
BASE_DIR=.
CONF_DIR=${BASE_DIR}/conf
OUTPUT_DIR=${BASE_DIR}/output
LOG_DIR=${BASE_DIR}/log

# configuration files
CMD_CONF=${CONF_DIR}/cmd.conf
HOST_CONF=${CONF_DIR}/hosts.conf

# remote username
USERNAME=root

# logfile (filename or '-' for stdout
#LOG_FILE=${LOG_DIR}/fetch-data.log
LOG_FILE=-

# temporary output file
TEMP_FILE=/tmp/stats.temp.output-$$.txt

# way to separate output data from different command executions
SEP="##################################"
SL=">>>>>>>> --"
SR="-- <<<<<<<<"

# regular expression to skip comment lines on configuration files
LINE_IGNORE="(^#.*|^ *$|^$)"

#
# calculates current time
# do_time generates time string like hh:mm:ss on $TIME
# do_date generates date string like yyyy-mm-dd on $TODAY
# do_now generates time string like yyyy-mm-dd hh:mm:ss on $NOW
#
# usage: do_time
# usage: do_date
# usage: do_now
function do_time() {
	TIME=$(${DATE} +"%T")
}
function do_date() {
	TODAY=$(${DATE} +"%F")
}
function do_now() {
	do_time
	do_date
	NOW="${TODAY} ${TIME}"
}

#
# check for files and config directories existence and permissions
#
# usage: check_dirs
#
function check_dirs() {
	echo -n "checking for necessary directories... "
	if [ ! -d ${BASE_DIR} ] || [ ! -x ${BASE_DIR} ]; then
		echo "unable to find or read ${BASE_DIR}"
		return 0
	fi
	if [ ! -d ${CONF_DIR} ] || [ ! -x ${CONF_DIR} ]; then
		echo "unable to find or read ${CONF_DIR}"
		return 0
	fi
	if [ ! -d ${OUTPUT_DIR} ] || [ ! -x ${OUTPUT_DIR} ]; then
		echo "unable to find or read ${OUTPUT_DIR}"
		return 0
	fi
	echo "ok"
	return 1
}

#
# check for configuration files existence and readability
#
# usage: check_config_files
#
function check_config_files() {
	echo -n "checking configuration files... "
	if [ ! -r ${CMD_CONF} ]; then
		echo "unable to find or read ${CMD_CONF}"
		return 0
	fi
	if [ ! -r ${HOSTS_CONF} ]; then
		echo "unable to find or read ${HOSTS_CONF}"
		return 0
	fi
	echo "ok"
	return 1
}

#
# check for log file writability
#
# usage: check_log_file
#
function check_log_files() {
	echo -n "checking log files... "
	if [ "${LOG_FILE}" == "-" ]; then
		echo "stdout:ok"
		return 1
	fi	
	if [ ! -d ${LOG_DIR} ] || [ ! -x ${LOG_DIR} ]; then
		echo "unable to find or read ${LOG_DIR}"
		return 0
	fi
	if [ ! -w ${LOG_FILE} ]; then
		echo "unable to find or write ${LOG_FILE}"
		return 0
	fi
	echo "${LOG_FILE}:ok"
	return 1
}

#
# check external binary dependecies
#
# usage: check_dep
#
function check_dep() {
	echo -n "checking dependencies... "
	if [ ! -x ${CAT} ]; then
		echo "unable to execute or find ${CAT}"
		return 0
	fi
	if [ ! -x ${SSH} ]; then
		echo "unable to execute or find ${SSH}"
		return 0
	fi
	if [ ! -x ${DATE} ]; then
		echo "unable to execute or find ${DATE}"
		return 0
	fi
	if [ ! -x ${CUT} ]; then
		echo "unable to execute or find ${CUT}"
		return 0
	fi
	if [ ! -x ${EGREP} ]; then
		echo "unable to execute or find ${EGREP}"
		return 0
	fi
	if [ ! -x ${RM} ]; then
		echo "unable to execute or find ${RM}"
		return 0
	fi
	if [ ! -x ${MKDIR} ]; then
		echo "unable to execute or find ${MKDIR}"
		return 0
	fi
	echo "ok"
	return 1
}

#
# check for directory existence and create it if necessary
#
# usage: new_dir directory_path
# return: 0 - ok, directory created; 1 - ok, directory already existed
#
new_dir()
{
        local DIR=${1}
        do_time
        echo -n "${TIME} - checking dir ${DIR}... "
        if [ ! -d ${DIR} ]; then
                OLD=0
                echo -n "creating... "
                ${MKDIR} -p ${DIR}
                echo -n "done: "
        else
                OLD=1
                echo -n "already exists: "
        fi

        # generate a return value: check for directory
        # existence and permissions
        if [ ! -d ${DIR} ] || [ ! -x ${DIR} ]; then
                echo "unable to use directory"
                return 0;
        else
                echo "ok"
                return $((${OLD}+1))
        fi
}

#
# Parse configuration files and generate command string to execute
# on each server
#
# usage: parse_cmd_config config_file
#
function parse_cmd_config() {
	local CONF_FILE=${1}
	# initialize $COMMAND variable; we want to start a command from scratch
	# (global)
	COMMAND=""

	do_time
	echo "${TIME} - parsing cmd configuration file (${CONF_FILE}):"
	while read LINE; do
		# field 1 is command
		local INPUT_COMMAND=$(echo $LINE | ${CUT} -d ':' -f 1)
		# field 2 is filename
		local INPUT_FILE=$(echo $LINE | ${CUT} -d ':' -f 2)	
		# field 3 is description
		local INPUT_DESC=$(echo $LINE | ${CUT} -d ':' -f 3)	
		echo "  * ${INPUT_DESC}: ${INPUT_COMMAND}"

		# cat new input command with global command line
		if [ "${COMMAND}" == "" ]; then
			COMMAND="echo \"${SEP} ${INPUT_FILE} ${SEP}\";${INPUT_COMMAND}"
		else
			COMMAND="${COMMAND};echo \"${SEP} ${INPUT_FILE} ${SEP}\";${INPUT_COMMAND}"
		fi
	done < <(${CAT} ${CONF_FILE} | ${EGREP} -v "${LINE_IGNORE}")
}

#
# Parse host configuration files and generate host
#
# usage: parse_host_config config_file
#
function parse_host_config() {
	local CONF_FILE=${1}
	do_time
	echo "${TIME} - parsing host configuration file (${CONF_FILE}):"
	while read LINE; do
		if [ "${HOSTS}" == "" ]; then
			HOSTS="${LINE}"
		else
			HOSTS="${HOSTS} ${LINE}"
		fi
		echo "  * ${LINE}"
	done < <(${CAT} ${CONF_FILE} | ${EGREP} -v "${LINE_IGNORE}")
}

#
# remotely execute commands on multiple servers
#
# usage: execute_remote $COMMAND $HOSTS 
#
function execute_remote() {
	local CMD=${1}
	local HOSTS=${2}

	for HOST in ${HOSTS}; do
		do_time
		echo "${TIME} - host: ${HOST}:"
		# maybe we should check if new_dir returned ok ...
		new_dir ${OUTPUT_DIR}/${HOST}

		echo -n "  * executing remote commands on ${HOST}... "
		${SSH} -n ${USERNAME}@${HOST} ${CMD} > ${TEMP_FILE} 2>&1
		echo "done"

		# set initial output file
		OUTPUT="/dev/null"
		echo -n "  * separating output from ${HOST} into configured data files... "
		# read from file descriptor 4 (or $TEMP_FILE)
		while read TEMP_LINE; do
			if [ ! -z "$(echo ${TEMP_LINE} | ${EGREP} "${SEP} .* ${SEP}")" ]; then
				# separation found; change $OUTPUT
				OUTPUT=${OUTPUT_DIR}/${HOST}/$(echo ${TEMP_LINE} | ${CUT} -d ' ' -f 2)
				# clear output file
				echo "" > ${OUTPUT}
			else
				echo ${TEMP_LINE} >> ${OUTPUT}
			fi
		done < <(${CAT} ${TEMP_FILE})
		echo "done"

		# clear temporary output file
		${RM} -f ${TEMP_FILE}

		# timestamp of last collected data
		do_now
		echo ${NOW} > ${OUTPUT_DIR}/${HOST}/timestamp
	done
}

if [ ! "${LOG_FILE}" == "-" ]; then
	# copy stdout into file descriptor 6
	exec 6>&1
	# redirect stdout into logfile
	exec 1>>${LOG_FILE}
fi

# start run log line
do_now
echo "${SL} fetch-data.sh start run at ${NOW} ${SR}"

# do all sorts of checks before startind doing real work
check_dep
if [ $? == 0 ]; then
	exit 1
fi
check_dirs
if [ $? == 0 ]; then
	exit 1
fi
check_config_files
if [ $? == 0 ]; then
	exit 1
fi
# check log files
check_log_files
if [ $? == 0 ]; then    
        exit 1
fi


# parse configuration file(s) and generate
# remote command string (global variable)
parse_cmd_config ${CMD_CONF}

# parse configuration file(s) and generate
# remote host string (global variable)
parse_host_config ${HOST_CONF}

# execute $COMMAND on each server
execute_remote "${COMMAND}" "${HOSTS}"

# end run log line
do_now
echo "${SL} fetch-data.sh end run   at ${NOW} ${SR}"