#!/bin/sh

#
# This file is released under the terms of the Artistic License.
# Please see the file LICENSE, included in this package, for details.
#
# Copyright The DBT-2 Authors
#

do_sleep()
{
	echo "sleeping $1 second(s)."
	sleep "${1}"
}

make_directories()
{
	COMMAND=""
	HOST=${1}
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	fi
	${COMMAND} mkdir -p "${OUTPUT_DIR}"
}

profile_collect()
{
	if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		which ts ts-profile > /dev/null 2>&1
		RC=$?
		if [ $RC -gt 1 ]; then
			echo "WARNING: touchstone tools not found for local profiling"
		else
			ts profile -o "${DRIVER_OUTPUT_DIR}/$(hostname)/profile" sleep 10 &
		fi
	fi

	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		eval "${CLIENT_COMMAND} which ts ts-profile" > /dev/null 2>&1
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for profiling %s\n" \
					"${HOSTNAME}"
		else
			eval "${CLIENT_COMMAND} ts profile \
					-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}/profile sleep 10 &"
		fi
	fi

	if [ $DBAAS -eq 0 ]; then
		HOSTNAME=$(eval "${DB_COMMAND} hostname")
		eval "${DB_COMMAND} which ts ts-profile" > /dev/null 2>&1
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for profiling %s\n" \
					"${HOSTNAME}"
		else
			eval "${DB_COMMAND} ts profile \
					-o ${DB_OUTPUT_DIR}/${HOSTNAME}/profile sleep 10 &"
		fi
	fi
}

profile_process()
{
	echo "processing profiling data..."
	if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		ts profile -o "${DRIVER_OUTPUT_DIR}/$(hostname)/profile" -p
	fi

	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		eval "${CLIENT_COMMAND} ts profile \
				-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}/profile -p"
	fi

	if [ $DBAAS -eq 0 ]; then
		HOSTNAME=$(eval "${DB_COMMAND} hostname")
		eval "${DB_COMMAND} ts profile \
				-o ${DB_OUTPUT_DIR}/${HOSTNAME}/profile -p"
	fi
}

stat_collection()
{
	if [ $STATS -ne 1 ]; then
		return
	fi

	ARGS=$1

	if [ "${CONFIGFILE}" = "" ]; then
		# If the this is a 1-tier test, system stats will be saved only in the
		# db directory.
		if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
			which ts ts-profile > /dev/null 2>&1
			RC=$?
			if [ $RC -gt 1 ]; then
				printf "WARNING: touchstone tools not found for driver system "
				echo "stats collection"
			else
				eval "ts sysstat -o ${DRIVER_OUTPUT_DIR}/$(hostname)/sysstat ${ARGS} &"
			fi
		fi

		# Handle system statistics on the Client node, if remote.
		if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
			HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
			eval "${CLIENT_COMMAND} which ts ts-profile" > /dev/null 2>&1
			RC=$?
			if [ $RC -gt 1 ]; then
				printf "WARNING: touchstone tools not found for local system "
				echo "stats collection"
			else
				eval "${CLIENT_COMMAND} ts sysstat \
						-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}/sysstat ${ARGS} &"
			fi
		fi

		# Don't collect system stats from the database system if the database
		# is a service.
		HOSTNAME=$(eval "${DB_COMMAND} hostname")
		if [ $DBAAS -eq 0 ]; then
			eval "${DB_COMMAND} which ts ts-sysstat" > /dev/null 2>&1
			RC=$?
			if [ $RC -gt 1 ]; then
				printf "WARNING: touchstone tools not found for system stats "
				printf "collection on %s\n" "${HOSTNAME}"
			else
				eval "${DB_COMMAND} ts sysstat \
						-o ${DB_OUTPUT_DIR}/${HOSTNAME}/sysstat ${ARGS} &"
			fi
		fi

		eval "${DB_COMMAND} which ts ts-${DBMS}-stat" > /dev/null 2>&1
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for database stats "
			printf "collection for RDBMS %s on system %s\n" \
					"${DBMS}" "${HOSTNAME}"
		else
			eval "${DB_COMMAND} ts ${DBMS}-stat \
					-o ${DB_OUTPUT_DIR}/${HOSTNAME}/dbstat ${DBSTATARGS} ${ARGS} &"
		fi
	else
		for SYSTEM in ${DRIVERLIST}; do
			if [ "${SYSTEM}" = "localhost" ]; then
				CMD=""
			else
				CMD="${SSH} ${SYSTEM}"
			fi
			HOSTNAME=$(eval "${CMD} hostname")
			if eval "${CMD} command -v ts" > /dev/null; then
				eval "${CMD} ts sysstat \
						-o ${DRIVER_OUTPUT_DIR}/${HOSTNAME}/sysstat ${ARGS} &"
			else
				printf "WARNING: touchstone tools not found for "
				printf "stats collection on system %s\n" "${HOSTNAME}"
			fi
		done
		for SYSTEM in ${CLIENTLIST}; do
			if [ "${SYSTEM}" = "localhost" ]; then
				CMD=""
			else
				CMD="${SSH} ${SYSTEM}"
			fi
			HOSTNAME=$(eval "${CMD} hostname")
			if eval "${CMD} command -v ts" > /dev/null; then
				eval "${CMD} ts sysstat \
						-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}/sysstat ${ARGS} &"
			else
				printf "WARNING: touchstone tools not found for "
				printf "stats collection on system %s\n" "${HOSTNAME}"
			fi
		done
		for SYSTEM in ${DBLIST}; do
			if [ "${SYSTEM}" = "localhost" ]; then
				CMD=""
			else
				CMD="${SSH} ${SYSTEM}"
			fi
			HOSTNAME=$(eval "${CMD} hostname")
			if eval "${CMD} command -v ts" > /dev/null; then
				eval "${CMD} ts sysstat \
						-o ${DB_OUTPUT_DIR}/${HOSTNAME}/sysstat ${ARGS} &"
				eval "${CMD} ts ${DBMS}-stat \
						-o ${DB_OUTPUT_DIR}/${HOSTNAME}/dbstat ${DBSTATARGS} \
						${ARGS} &"
			else
				printf "WARNING: touchstone tools not found for "
				printf "stats collection on system %s\n" "${HOSTNAME}"
			fi
		done
	fi
}

stop_processes() {
	if [ "${CONFIGFILE}" = "" ]; then
		find "${OUTPUT_DIR}" -type f -name dbt2_driver.pid 2> /dev/null |
				while IFS= read -r PIDFILE; do
			DRIVERPID=$(cat "${PIDFILE}")
			kill -9 "${DRIVERPID}"
		done
		if [ ! "$DRIVER3" = "1" ]; then
			if [ "${CLIENT_HOSTNAME}" = "localhost" ]; then
				CMD=""
			else
				CMD="${SSH} ${SYSTEM}"
			fi
			PIDFILE=$(eval "${CMD} find ${OUTPUT_DIR} -name dbt2_client.pid" \
					2> /dev/null)
			CLIENTPID=$(eval "${CMD} cat ${PIDFILE}")
			eval "${CMD} kill -9 ${CLIENTPID}"
		fi

		if [ ${PRIVILEGED} -eq 1 ]; then
			dbt2 "${DBMS}-stop-db"
		fi
	else
		for SYSTEM in ${CLIENTLIST}; do
			if [ "${SYSTEM}" = "localhost" ]; then
				CMD=""
			else
				CMD="${SSH} ${SYSTEM}"
			fi
			for PIDFILE in $(eval "${CMD} find ${OUTPUT_DIR} \
					-name dbt2_client.pid" 2> /dev/null); do
				CLIENTPID=$(eval "${CMD} cat ${PIDFILE}")
				eval "${CMD} kill -9 ${CLIENTPID}"
			done
		done
		for SYSTEM in ${DRIVERLIST}; do
			if [ "${SYSTEM}" = "localhost" ]; then
				CMD=""
			else
				CMD="${SSH} ${SYSTEM}"
			fi
			for PIDFILE in $(eval "${CMD} find ${OUTPUT_DIR} \
					-name dbt2_driver.pid" 2> /dev/null); do
				DRIVERPID=$(eval "${CMD} cat ${PIDFILE}")
				eval "${CMD} kill -9 ${DRIVERPID}"
			done
		done
	fi

	# Stop stat collection.
	stat_collection -s
}

trap 'printf "\n***Interrupted by Control-C.***\n\n"; stop_processes' INT
trap 'printf "\n***Interrupted by TERM signal.***\n\n"; stop_processes' TERM

usage()
{
	if [ "$1" != "" ]; then
		echo
		echo "error: $1"
		echo
	fi
	cat << EOF
$(basename "${0}") is the Database Test 2 (DBT-2) workload runner

Usage:
  $(basename "${0}") [OPTIONS] DBMS DIRECTORY

General options:
  -A, --advanced
                 enable advanced benchmarking execution options
  --comment=COMMENTS
                 provide COMMENTS to save with the test
  --config=CONFIG
                 CONFIG file to use for executing a test.  These settings will
                 override any conflicting command line arguments.
  --connection-delay=MILLISECONDS
                 MILLISECONDS delay between client and database connections,
                 default 100
  -d SECONDS, --duration=SECONDS
                 SECONDS of the test duration, after ramp up, default $DURATION
  --db-host=ADDRESS
                 ADDRESS of database system, default localhost
  --db-name=NAME
                 NAME of the database, default $DBNAME
  --db-port=PORT
                 database listening PORT number
  --db-user=USER
                 database USER
  --dbaas        database server is a service
  --districts=DISTRICTS
                 number of DISTRICTS per warehouse, default 10
  --driver-mode=MODE
                 1 for threaded, 3 for event-driven process MODE, default 3
  --profile      enable software profiling collection
  --privileged   run test as a privileged operating system and database user
  --stats        collect system and database stats
  -V, --version  output version information, then exit
  -w WAREHOUSES, --warehouses=WAREHOUSES
                 number of WAREHOUSES to use in the database, default $WAREHOUSES
  -?, --help     show this help, then exit

driver-mode 1 (threaded) specific options:
  -c CONNECTIONS, --client-connections=CONNECTIONS
                 number of database CONNECTIONS to open, default $DBCON
  --client-host=ADDRESS
                 ADDRESS of client system, default localhost
  --driver-partition-size=NUMBER
                 NUMBER of warehouses per process, default all
  --terminal-limit=TERMINALS
                 limit the number of TERMINALS emulated

driver-mode 3 (event-driven, process based) specific options:
  --connections-per-processor=PROCESSES
                 driver PROCESSES started per processor, default 1
  --terminal-limit=CONNECTIONS
                 limit the number of database CONNECTIONS opened

MySQL specific options:
  --db-password=PASSWORD
                 database PASSWORD
  --mysql-socket=FILE
                 specify socket FILE

PostgreSQL specific options:
  --db-parameters=OPTIONS
                 GUC command line OPTIONS to pass to postgres, when running
                 by privileged user

DBMS options are:
  cockroach  CockroachDB
  mysql      MySQL
  pgsql      PostgreSQL
  sqlite     SQLite

DIRECTORY is the path to save test results.

Database Test 2 (DBT-2) project page: https://github.com/osdldbt/dbt2
EOF
}

validate_parameter()
{
	if [ "$2" != "$3" ]; then
		usage "wrong argument '$2' for parameter '$1'"
		exit 1
	fi
}

# Tell ssh not to check known hosts or host keys.
SSH="ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

ALTERED_ARG=""
CLIENT_HOSTNAME="localhost"
CLIENTLIST=""
CONFIGFILE=""
DB_HOSTNAME="localhost"
DB_PASSWORD=""
DBLIST=""
DBAAS=0
DBCON=10
DBNAME="dbt2"
DRIVER3=1
DRIVERLIST=""
DURATION=180
EASYFLAGS="-ktd 0 -ktn 0 -kto 0 -ktp 0 -kts 0 -ttd 0 -ttn 0 -tto 0 -ttp 0 -tts 0"
EXTRA_DB_ARGS=""
FPP=1
PRIVILEGED=0
PROFILE=0
STATS=0
THREAD_STARTUP_DELAY=100 # milliseconds
THREADS_PER_WAREHOUSE=10
WAREHOUSES=1

# Custom argument handling for hopefully most portability.
while [ "${#}" -gt 0 ] ; do
	opt=${1}
	case "${1}" in
	(-A | --advanced)
		# Advanced benchmarking usage.  Disable all "easy" flags.
		EASYFLAGS=""
		;;
	(-c | --client-connections)
		shift
		# Check for numeric value
		DBCON=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DBCON}"
		;;
	(-c?*)
		# Check for numeric value
		DBCON=$(echo "${1#*-c}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DBCON}"
		;;
	(--client-connections=?*)
		# Check for numeric value
		DBCON=$(echo "${1#*--client-connections=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DBCON}"
		;;
	(--config)
		shift
		CONFIGFILE="${1}"
		;;
	(--config=*)
		CONFIGFILE="${1#*--config=}"
		;;
	(--client-host)
		shift
		CLIENT_HOSTNAME=$1
		;;
	(--client-host=?*)
		CLIENT_HOSTNAME=${1#*--client-host=}
		;;
	(--comment)
		shift
		COMMENT=$1
		;;
	(--comment=?*)
		COMMENT=${1#*--comment=}
		;;
	(--connection-delay)
		shift
		THREAD_STARTUP_DELAY=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${THREAD_STARTUP_DELAY}"
		;;
	(--connection-delay=?*)
		THREAD_STARTUP_DELAY=$(echo "${1#*--connection-delay=}" \
				| grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--connection-delay=}" \
				"${THREAD_STARTUP_DELAY}"
		;;
	(--connections-per-processor)
		shift
		FPP="$1"
		FPP_FLAG="-fpp $FPP"
		;;
	(--connections-per-processor=?*)
		FPP="${1#*--connections-per-processor=}"
		FPP_FLAG="-fpp $FPP"
		;;
	(-d | --duration)
		shift
		DURATION=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DURATION}"
		;;
	(-d?*)
		DURATION=$(echo "${1#*-d}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*-d}" "${DURATION}"
		;;
	(--duration=?*)
		DURATION=$(echo "${1#*--duration=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--duration=}" "${DURATION}"
		;;
	(--db-host)
		shift
		DB_HOSTNAME=${1}
		;;
	(--db-host=?*)
		DB_HOSTNAME=${1#*--db-host=}
		;;
	(--db-name)
		shift
		DBNAME=${1}
		;;
	(--db-name=?*)
		DBNAME=${1#*--db-name=}
		;;
	(--db-parameters)
		shift
		DB_PARAMS="${1}"
		DB_PARAMS_ARG="-p \"${DB_PARAMS}\""
		;;
	(--db-parameters=?*)
		DB_PARAMS="${1#*--db-parameters=}"
		DB_PARAMS_ARG="-p \"${DB_PARAMS}\""
		;;
	(--db-password)
		shift
		DB_PASSWORD=${1}
		;;
	(--db-password=?*)
		DB_PASSWORD=${1#*--db-password=}
		;;
	(--db-port)
		shift
		DB_PORT=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DB_PORT}"
		;;
	(--db-port=?*)
		DB_PORT=$(echo "${1#*--db-port=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--db-port=}" "${DB_PORT}"
		;;
	(--db-user)
		shift
		DB_USER=${1}
		;;
	(--db-user=?*)
		DB_USER=${1#*--db-user=}
		;;
	(--dbaas)
		DBAAS=1
		;;
	(--districts)
		shift
		THREADS_PER_WAREHOUSE=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${THREADS_PER_WAREHOUSE}"
		;;
	(--districts=?*)
		THREADS_PER_WAREHOUSE=$(echo "${1#*--districts=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--districts=}" \
				"${THREADS_PER_WAREHOUSE}"
		;;
	(--driver-mode)
		shift
		# Logic is pretty basic at the moment, set to 3 or unset it.
		if [ "${1}" -ne 3 ]; then
			DRIVER3=""
		fi
		;;
	(--driver-mode=?*)
		# Logic is pretty basic at the moment, set to 3 or unset it.
		if [ "${1#*--driver-mode=}" -ne 3 ]; then
			DRIVER3=""
		fi
		;;
	(--driver-partition-size)
		shift
		# Number of warehouses per driver.
		W_CHUNK=${1}
		;;
	(--driver-partition-size=?*)
		# Number of warehouses per driver.
		W_CHUNK=${1#*--driver-partition-size=}
		;;
	(--mysql-socket)
		shift
		EXTRA_DB_ARGS="${EXTRA_DB_ARGS} -S ${1}"
		;;
	(--mysql-socket=?*)
		EXTRA_DB_ARGS="${EXTRA_DB_ARGS} -S ${1#*--mysql-socket=}"
		;;
	(--privileged)
		PRIVILEGED=1
		;;
	(--profile)
		PROFILE=1
		;;
	(-S)
		shift
		SOCK_FILE_ARG="-t ${1}"
		;;
	(-S?*)
		SOCK_FILE_ARG="-t ${1#*-S}"
		;;
	(--stats)
		STATS=1
		;;
	(--terminal-limit)
		shift
		TERMINALS_LIMIT="$1"
		TERMINALS_LIMIT_ARG="-L ${TERMINALS_LIMIT}"
		;;
	(--terminal-limit=?*)
		TERMINALS_LIMIT="${1#*--terminal-limit=}"
		TERMINALS_LIMIT_ARG="-L ${TERMINALS_LIMIT}"
		;;
	(-V | --version)
		echo "$(basename "${0}") v0.61.6"
		exit 0
		;;
	(-w | --warehouses)
		shift
		WAREHOUSES=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${WAREHOUSES}"
		;;
	(-w?*)
		WAREHOUSES=$(echo "${1#*-w}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*-w}" "${WAREHOUSES}"
		;;
	(--warehouses=?*)
		WAREHOUSES=$(echo "${1#*--warehouses=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--warehouses=}" "${WAREHOUSES}"
		;;
	(-\? | --help)
		usage
		exit 0
		;;
	(--* | -*)
		echo "$(basename "${0}"): invalid option -- '${1}'"
		echo "try \"$(basename "${0}") --help\" for more information."
		exit 1
		;;
	(*)
		break
		;;
	esac
	shift
done

if [ $# -eq 0 ]; then
	printf "Specify which DBMS system to test, try \"%s --help\" " \
			"$(basename "${0}")"
	echo "for more information."
	exit 1
fi

DBMS=$1
shift

if [ $# -eq 0 ]; then
	echo "Specify the location to save test results, where directory does "
	printf "not exist yet, try \"%s\" " "$(basename "${0}")"
	echo "for more information."
	exit 1
fi

OUTPUT_DIR="${1}"

if [ -d "${OUTPUT_DIR}" ]; then
	echo "Directory '${OUTPUT_DIR}' exists, stopping to prevent possible"
	echo "clobbering of data, please specify another location."
	exit 1
fi

CLIENT_OUTPUT_DIR=${OUTPUT_DIR}/client
DRIVER_OUTPUT_DIR=${OUTPUT_DIR}/driver
DB_OUTPUT_DIR=${OUTPUT_DIR}/db
mkdir -p "${OUTPUT_DIR}" "${CLIENT_OUTPUT_DIR}" "${DRIVER_OUTPUT_DIR}" \
		"${DB_OUTPUT_DIR}"

SUMMARY="${OUTPUT_DIR}/summary.rst"

if [ ! "${DB_PORT}" = "" ]; then
	if [ "$DRIVER3" = "1" ]; then
		DB_PORT_ARG="-P ${DB_PORT}"
	else
		DB_PORT_ARG="-l ${DB_PORT}"
	fi
fi

DBSTATARGS=""
case $DBMS in
mysql)
	if [ ! "${DB_USER}" = "" ]; then
		EXTRA_DB_ARGS="${EXTRA_DB_ARGS} -u ${DB_USER}"
	fi
	;;
pgsql)
	# Set extra argument variables.
	DBSTATARGS="-h $DB_HOSTNAME -d $DBNAME"
	if [ ! "${DB_USER}" = "" ]; then
		DBSTATARGS="${DBSTATARGS} -U ${DB_USER}"
	fi
	if [ ! "${DB_PORT}" = "" ]; then
		DBSTATARGS="${DBSTATARGS} -p ${DB_PORT}"
	fi
	;;
esac

if [ "${CONFIGFILE}" = "" ]; then
	# If no chunking of warehouses per process is defined, assumed we want to run
	# everything from one process.
	if [ "${W_CHUNK}" = "" ]; then
		W_CHUNK=${WAREHOUSES}
	fi

	# Check parameters.

	if [ "${THREADS_PER_WAREHOUSE}" -lt 1 ] || \
			[ "${THREADS_PER_WAREHOUSE}" -gt 1000 ];
	then
		usage "--districts value should be in range [1..1000]. Please specify correct value"
		exit 1
	fi

    # Since this code path is running the driver on the current system, we will
    # have all the data to calculate the primary metric.
	if ! which rsync > /dev/null 2>&1; then
		printf "WARNING: rsync required on current system in order to "
        echo "retrieve data from any remote systems"
	fi

	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		CLIENT_COMMAND="$SSH $CLIENT_HOSTNAME"
		if ! eval "${CLIENT_COMMAND} which rsync" > /dev/null 2>&1; then
			printf "WARNING: rsync required on client system in order to "
			echo "retrieve client data"
		fi
	else
		CLIENT_COMMAND=""
	fi

	if [ ${STATS} -eq 1 ] && [ ! "${DB_HOSTNAME}" = "localhost" ] \
            && [ $DBAAS -eq 0 ]; then
		DB_COMMAND="$SSH ${DB_HOSTNAME}"
		if ! eval "${DB_COMMAND} which rsync" > /dev/null 2>&1; then
			printf "WARNING: rsync required on database system in order to "
			echo "retrieve database data"
		fi
	else
		DB_COMMAND=""
	fi

	#
	# Create the directories we will need.
	#
	make_directories
	mkdir -p "${DRIVER_OUTPUT_DIR}/$(hostname)"
	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		make_directories "${CLIENT_HOSTNAME}"
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		eval "${CLIENT_COMMAND} mkdir -p ${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
	fi
	if [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		#
		# Create direcotires on the database server if remote.
		#
	    if [ ${STATS} -eq 1 ] && [ $DBAAS -eq 0 ]; then
			make_directories "${DB_HOSTNAME}"
			$SSH "$DB_HOSTNAME" -- mkdir -p "$DB_OUTPUT_DIR"
		fi
	fi
	mkdir -p "${DB_OUTPUT_DIR}"
else
	if [ ! -f "${CONFIGFILE}" ]; then
		echo "ERROR: config file '${CONFIGFILE}' not found"
		exit 98
	fi

	DBNAME="$(toml get "${CONFIGFILE}" . | jq -r '.database_name')"
	DURATION="$(toml get "${CONFIGFILE}" . | jq -r '.duration')"
	WAREHOUSES="$(toml get "${CONFIGFILE}" . | jq -r '.warehouses')"

	TMP="$(toml get "${CONFIGFILE}" . | jq -r '.mode')"
	if [ "${TMP}" = "3" ]; then
		DRIVER3=1
	else
		DRIVER3=0
	fi
fi

# Enable altered execution mode only if DRIVER3 is enabled.
if [ "${DRIVER3}" = "1" ]; then
	ALTERED_ARG="-altered ${DRIVER3}"
fi

# Create a readme file in the output directory, date it and add run information
# into the readme.txt.
cat >> "${OUTPUT_DIR}/readme.txt" << EOF
$(date)
$COMMENT
Command line: $0 $*
RDBMS: $DBMS
Database Name: $DBNAME
Database Scale Factor: $WAREHOUSES warehouses
Test Duration: $DURATION seconds
EOF

if [ "${CONFIGFILE}" = "" ]; then
	# Get any OS specific information, if this is a single tier configuration,
	# then the OS parameters will be in the database output directory.
	if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		dbt2 get-os-info -o "${DRIVER_OUTPUT_DIR}/$(hostname)"
		HOSTNAME=$(eval "${DB_COMMAND} hostname")
		eval "${DB_COMMAND} dbt2 get-os-info -o ${DB_OUTPUT_DIR}/${HOSTNAME}"
	else
		dbt2 get-os-info -o "${DB_OUTPUT_DIR}/$(hostname)"
	fi

	if [ $PRIVILEGED -eq 1 ]; then
		if eval "${DB_COMMAND} dbt2 ${DBMS}-stop-db ${MISC_ARG}"; then
			echo "cleaning up results..."
			rm -rf "${OUTPUT_DIR}"
			exit 1
		fi
		eval "${DB_COMMAND} dbt2 ${DBMS}-start-db ${DB_PARAMS_ARG} ${MISC_ARG} \
				-o ${DB_OUTPUT_DIR}" || exit 1
	fi
fi

#
# Redisplay the test parameters.
#
echo "# DBT-2 test for ${DBMS} started at $(date)"
echo
echo "Parameters:"
echo
echo "* SCALE FACTOR (WAREHOUSES): ${WAREHOUSES}"
echo "* DURATION OF TEST (seconds): ${DURATION}"
if [ ! "${DBNAME}" = "" ]; then
	echo "* DATABASE NAME: ${DBNAME}"
fi
echo "* TERMINALS PER WAREHOUSE: ${THREADS_PER_WAREHOUSE}"

#
# Build up the client command line arguments.
#

CLIENT_COMMAND_ARGS="-a $DBMS"

if [ -n "${DB_USER}" ]; then
	echo "* DATABASE USER: ${DB_USER}"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -u ${DB_USER}"
fi

if [ -n "${DB_PASSWORD}" ]; then
	echo "* DATABASE PASSWORD: *******"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -a ${DB_PASSWORD}"
fi

if [ "${CONFIGFILE}" = "" ]; then
	if [ -n "${DB_SOCKET}" ]; then
		echo "* DATABASE SOCKET: ${DB_SOCKET}"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -t ${DB_SOCKET}"
	fi

	if [ -n "${DB_PORT}" ]; then
		echo "* DATABASE PORT: ${DB_PORT}"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} ${DB_PORT_ARG}"
	fi

	OPTION1=$(( WAREHOUSES * THREADS_PER_WAREHOUSE ))
	THREADS=$OPTION1
	if [ ! "$TERMINALS_LIMIT" = "" ]; then
		OPTION2=$TERMINALS_LIMIT
		if [ "${OPTION1}" -lt "${OPTION2}" ]; then
			THREADS=$OPTION1
		else
			THREADS=$OPTION2
		fi
	fi
	if [ "$DRIVER3" = "1" ]; then
		echo "* DATABASE CONNECTIONS: $(nproc)"
		echo "* TERMINAL PROCESSES: $(nproc)"
	else
		echo "* DATABASE CONNECTIONS: ${DBCON}"
		echo "* TERMINAL THREADS: ${THREADS}"
	fi
	if [ ! "$DRIVER3" = "1" ]; then
		echo "* TERMINAL CAP: $TERMINALS_LIMIT"
	fi
	echo "* WAREHOUSES PER DRIVER/CLIENT PAIR: $W_CHUNK"
fi
echo

#
# Start the client.
#
echo "## Stage 1. Starting up client..."
echo
if [ "${CONFIGFILE}" = "" ]; then
	if [ ! "$DRIVER3" = "1" ]; then
		if [ "${DBMS}" = "cockroach" ]; then
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_HOSTNAME}"
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -b ${DBNAME}"
		elif [ "${DBMS}" = "mysql" ]; then
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DBNAME}"
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} ${SOCK_FILE_ARG} -u root"
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -h ${DB_HOSTNAME}"
		elif [ "${DBMS}" = "pgsql" ]; then
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_HOSTNAME}"
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -b ${DBNAME}"
		elif [ "${DBMS}" = "sqlite" ]; then
			CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_PARAMS}"
		fi
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -f -c $DBCON"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -s ${THREAD_STARTUP_DELAY}"
		END=0
		START=1

		#
		# Count of the number of clients to run.
		#
		SEG=0
		PORT=30000
		while [ "${END}" -lt "${WAREHOUSES}" ]; do
			START=$(( END + 1 ))
			END=$(( START + W_CHUNK ))
			END=$(( END - 1 ))
			SEG=$(( SEG + 1 ))

			if [ "${END}" -gt "${WAREHOUSES}" ]; then
				END=${WAREHOUSES}
			fi

			HOSTNAME=$($CLIENT_COMMAND hostname)
			CPDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
			mkdir -p "${CPDIR}"

			CDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}.${SEG}"
			eval "${CLIENT_COMMAND} mkdir -p ${CDIR}"

			CARGS="${CLIENT_COMMAND_ARGS} -p ${PORT} -o ${CDIR}"
			eval "${CLIENT_COMMAND} dbt2 client ${CARGS} \
					> ${CPDIR}/client-${SEG}.txt 2>&1 &"

			PORT=$(( PORT + 1))
		done

		# Sleep long enough for all the client database connections to be
		# established.
		CLIENT_RAMPUP_TIME=$DBCON
		CLIENT_RAMPUP_TIME=$(( CLIENT_RAMPUP_TIME * THREAD_STARTUP_DELAY ))
		CLIENT_RAMPUP_TIME=$(( CLIENT_RAMPUP_TIME / 1000 ))
		CLIENT_RAMPUP_TIME=$(( CLIENT_RAMPUP_TIME + 1 ))

		printf "Waiting for clients to ramp up, "
		do_sleep $CLIENT_RAMPUP_TIME
	else
		echo "No clients to start."
	fi

	# Calculate how many terminals are being emulated.
	OPTION1=$(( W_CHUNK + 1 ))
	OPTION1=$(( OPTION1 * THREADS_PER_WAREHOUSE ))
	DRIVER_RAMPUP_TIME=$OPTION1
	if [ ! "$TERMINALS_LIMIT" = "" ]; then
		OPTION2=$TERMINALS_LIMIT
		if [ "${OPTION1}" -lt "${OPTION2}" ]; then
			DRIVER_RAMPUP_TIME=$OPTION1
		else
			DRIVER_RAMPUP_TIME=$OPTION2
		fi
	fi
	if [ "$DRIVER3" = "1" ]; then
		DRIVER_RAMPUP_TIME=$(( ($(nproc) * FPP) - 1 ))
		if [ ! "$TERMINALS_LIMIT" = "" ] &&
				[ $TERMINALS_LIMIT -lt $DRIVER_RAMPUP_TIME ]; then
			DRIVER_RAMPUP_TIME="${TERMINALS_LIMIT}"
		fi
	fi
	DRIVER_RAMPUP_TIME=$(( DRIVER_RAMPUP_TIME * THREAD_STARTUP_DELAY ))
	DRIVER_RAMPUP_TIME=$(( DRIVER_RAMPUP_TIME / 1000 ))
	TOTAL_RUN_TIME=$(( DRIVER_RAMPUP_TIME + DURATION ))
	SAMPLE_LENGTH=60
	ITERATIONS=$(( TOTAL_RUN_TIME / SAMPLE_LENGTH ))
	ITERATIONS=$(( ITERATIONS + 1 ))

	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		eval "${CLIENT_COMMAND} dbt2 get-os-info \
				-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
	fi

	HOSTNAME=$(eval "${DB_COMMAND} hostname")
	eval "$DB_COMMAND mkdir -p ${DB_OUTPUT_DIR}/${HOSTNAME}"
	if [ $DBAAS -eq 0 ]; then
		eval "${DB_COMMAND} dbt2 ${DBMS}-plans -d ${DBNAME} \
				-o ${DB_OUTPUT_DIR}/${HOSTNAME}/plan0.txt"
	fi
else
	if [ ! "${DRIVER3}" = "1" ]; then
		CLIENTS="$(toml get "${CONFIGFILE}" . | jq -r '.client | length')"

		printf "Starting up %d client(s):\n\n" "${CLIENTS}"

		CLIENT_RAMPUP_TIME=0
		for INDEX in $(seq 0 $(( CLIENTS - 1 ))); do
			CLIENT_HOSTNAME="$(toml get "${CONFIGFILE}" client | \
					jq -r ".[${INDEX}].client_addr")"
			DB_HOSTNAME="$(toml get "${CONFIGFILE}" client | \
					jq -r ".[${INDEX}].database_addr")"
			DBCON="$(toml get "${CONFIGFILE}" client | \
					jq -r ".[${INDEX}].connections")"

			TMP="$(toml get "${CONFIGFILE}" client | \
					jq -r ".[${INDEX}].connection_delay")"
			if [ ! "${TMP}" = "null" ]; then
				THREAD_STARTUP_DELAY="${TMP}"
			fi

			PORT=30000
			TMP="$(toml get "${CONFIGFILE}" client | \
					jq -r ".[${INDEX}].client_port")"
			if [ ! "${TMP}" = "null" ]; then
				PORT="${TMP}"
			fi

			printf "%d. %s listening on %d with %d connection(s) to %s.\n" \
					$(( INDEX + 1 )) "${CLIENT_HOSTNAME}" "${PORT}" "${DBCON}" \
					"${DB_HOSTNAME}"

			CLIENTLIST="${CLIENTLIST} ${CLIENT_HOSTNAME}"
			DBLIST="${DBLIST} ${DB_HOSTNAME}"

			CLIENT_CMD_ARGS2="${CLIENT_COMMAND_ARGS}"
			if [ "${DBMS}" = "cockroach" ]; then
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -d ${DB_HOSTNAME}"
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -b ${DBNAME}"
			elif [ "${DBMS}" = "mysql" ]; then
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -d ${DBNAME}"
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} ${SOCK_FILE_ARG} -u root"
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -h ${DB_HOSTNAME}"
			elif [ "${DBMS}" = "pgsql" ]; then
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -d ${DB_HOSTNAME}"
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -b ${DBNAME}"
			elif [ "${DBMS}" = "sqlite" ]; then
				CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -d ${DB_PARAMS}"
			fi
			CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -f -c ${DBCON}"
			CLIENT_CMD_ARGS2="${CLIENT_CMD_ARGS2} -s ${THREAD_STARTUP_DELAY}"

			if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
				CLIENT_COMMAND="${SSH} ${CLIENT_HOSTNAME}"
				if ! eval "${CLIENT_COMMAND} which rsync" > /dev/null 2>&1;
				then
					printf "WARNING: rsync required on client system in order "
					echo "to retrieve data from client ${CLIENT_HOSTNAME}"
				fi
			else
				CLIENT_COMMAND=""
			fi

			HOSTNAME="$(${CLIENT_COMMAND} hostname)"
			CPDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
			mkdir -p "${CPDIR}"

			CDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}.${INDEX}"
			eval "${CLIENT_COMMAND} mkdir -p ${CDIR}"

			CARGS="${CLIENT_CMD_ARGS2} -p ${PORT} -o ${CDIR}"
			eval "${CLIENT_COMMAND} dbt2 client ${CARGS} \
					> ${CPDIR}/client-${INDEX}.txt 2>&1 &"

			# Sleep for only the duration of the client with the longest ramp
			# up time.
			THIS_RAMPUP_TIME=${DBCON}
			THIS_RAMPUP_TIME=$(( THIS_RAMPUP_TIME * THREAD_STARTUP_DELAY ))
			THIS_RAMPUP_TIME=$(( THIS_RAMPUP_TIME / 1000 ))
			THIS_RAMPUP_TIME=$(( THIS_RAMPUP_TIME + 1 ))

			if [ ${THIS_RAMPUP_TIME} -gt ${CLIENT_RAMPUP_TIME} ]; then
				CLIENT_RAMPUP_TIME=${THIS_RAMPUP_TIME}
			fi
		done

		CLIENTLIST="$(echo "${CLIENTLIST}" | \
				awk -v RS='[[:space:]]+' '!a[$0]++{printf "%s%s", $0, RT}')"

		echo
		printf "Client systems: %s\n" "${CLIENTLIST}"
		echo
		printf "Waiting for clients to ramp up, "
		do_sleep $CLIENT_RAMPUP_TIME
	else
		echo "No clients to start."
	fi
fi
echo

# Start the driver.
echo "## Stage 2. Starting up driver..."
echo

if [ "${CONFIGFILE}" = "" ]; then
	END=0
	START=1
	#
	# Count of the number of drivers run.
	#
	SEG=0
	PORT=30000
	while [ "${END}" -lt "${WAREHOUSES}" ]; do
		START=$(( END + 1 ))
		END=$(( START + W_CHUNK ))
		END=$(( END - 1 ))
		SEG=$(( SEG + 1 ))

		if [ "${END}" -gt "${WAREHOUSES}" ]; then
			END=${WAREHOUSES}
		fi

		DDIR="${DRIVER_OUTPUT_DIR}/$(hostname).${SEG}"
		mkdir -p "${DDIR}"
		DRIVER_COMMAND_ARGS="-l $DURATION -wmin $START"
		DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -wmax $END -w $WAREHOUSES"
		DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -sleep $THREAD_STARTUP_DELAY"
		DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -outdir $DDIR"
		DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -tpw $THREADS_PER_WAREHOUSE"
		DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS $EASYFLAGS $TERMINALS_LIMIT_ARG"
		DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS $ALTERED_ARG $DB_PORT_ARG"
		if [ "$DRIVER3" = "1" ]; then
			DHNARG="-b ${DBNAME} -d ${DB_HOSTNAME} ${EXTRA_DB_ARGS}"
			eval "dbt2 driver3 $DRIVER_COMMAND_ARGS -a $DBMS $DHNARG $FPP_FLAG \
					> ${DRIVER_OUTPUT_DIR}/$(hostname)/driver-${SEG}.txt 2>&1" \
					|| exit 1 &
		else
			eval "dbt2 driver -d $CLIENT_HOSTNAME -p $PORT $DRIVER_COMMAND_ARGS > \
					${DRIVER_OUTPUT_DIR}/$(hostname)/driver-${SEG}.txt 2>&1" \
					|| exit 1 &
		fi
		DRIVER_PIDLIST="$DRIVER_PIDLIST $!"
		PORT=$(( PORT + 1 ))
	done
else
	DRIVERS="$(toml get "${CONFIGFILE}" . | jq -r '.driver | length')"

	printf "Starting up %d driver(s):\n\n" "${DRIVERS}"

	DRIVER_RAMPUP_TIME=0
	START=1
	END="${WAREHOUSES}"
	for INDEX in $(seq 0 $(( DRIVERS - 1 ))); do
		FPP_FLAG=""

		DRIVER_HOSTNAME="$(toml get "${CONFIGFILE}" driver | \
				jq -r ".[${INDEX}].driver_addr")"

		TMP="$(toml get "${CONFIGFILE}" driver | \
				jq -r ".[${INDEX}].connection_limit")"
		if [ ! "${TMP}" = "null" ]; then
			TERMINALS_LIMIT="${TMP}"
			TERMINALS_LIMIT_ARG="-L ${TERMINALS_LIMIT}"
		fi

		TMP="$(toml get "${CONFIGFILE}" driver | \
				jq -r ".[${INDEX}].connection_delay")"
		if [ ! "${TMP}" = "null" ]; then
			THREAD_STARTUP_DELAY="${TMP}"
		fi

		TMP="$(toml get "${CONFIGFILE}" driver | \
				jq -r ".[${INDEX}].connections_per_processor")"
		if [ ! "${TMP}" = "null" ]; then
			FPP="${TMP}"
			FPP_FLAG="-fpp ${FPP}"
		fi

		TMP="$(toml get "${CONFIGFILE}" driver | jq -r ".[${INDEX}].wmin")"
		if [ ! "${TMP}" = "null" ]; then
			START="${TMP}"
		fi

		TMP="$(toml get "${CONFIGFILE}" driver | jq -r ".[${INDEX}].wmax")"
		if [ ! "${TMP}" = "null" ]; then
			END="${TMP}"
		fi

		DRIVERLIST="${DRIVERLIST} ${DRIVER_HOSTNAME}"

		if [ "${DRIVER_HOSTNAME}" = "localhost" ]; then
			DRIVER_COMMAND=""
			HOSTNAME="$(hostname)"
		else
			DRIVER_COMMAND="${SSH} ${DRIVER_HOSTNAME}"
			if ! eval "${DRIVER_COMMAND} which rsync" > /dev/null 2>&1; then
				printf "ERROR: rsync required on driver system in order to "
				echo "calculate primary metric: ${DRIVER_HOSTNAME}"
				exit 1
			fi
			HOSTNAME="$(eval "${DRIVER_COMMAND} hostname")"
		fi

		DDIR="${DRIVER_OUTPUT_DIR}/${HOSTNAME}.${INDEX}"
		eval "${DRIVER_COMMAND} mkdir -p ${DDIR} \
				${DRIVER_OUTPUT_DIR}/${HOSTNAME}"

		DRIVER_CMD_ARGS="-l ${DURATION}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} -wmin ${START}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} -wmax ${END}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} -w ${WAREHOUSES}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} -sleep ${THREAD_STARTUP_DELAY}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} -outdir ${DDIR}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} -tpw ${THREADS_PER_WAREHOUSE}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} ${EASYFLAGS}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} ${TERMINALS_LIMIT_ARG}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} ${ALTERED_ARG}"
		DRIVER_CMD_ARGS="${DRIVER_CMD_ARGS} ${DB_PORT_ARG}"

		if [ "${DRIVER3}" = "1" ]; then
			DB_HOSTNAME="$(toml get "${CONFIGFILE}" driver | \
					jq -r ".[${INDEX}].database_addr")"

			printf "%d. %s to database on %s, warehouses %d to %d.\n" \
					$(( INDEX + 1 )) "${DRIVER_HOSTNAME}" "${DB_HOSTNAME}" \
					"${START}" "${END}"

			eval "${DRIVER_COMMAND} dbt2 driver3 ${DRIVER_CMD_ARGS} \
					-a ${DBMS} -b ${DBNAME} -d ${DB_HOSTNAME} ${FPP_FLAG} > \
					${DRIVER_OUTPUT_DIR}/${HOSTNAME}/driver-${INDEX}.txt 2>&1" \
					|| exit 1 &
			DBLIST="${DBLIST} ${DB_HOSTNAME}"
		else
			CLIENT_HOSTNAME="$(toml get "${CONFIGFILE}" driver | \
					jq -r ".[${INDEX}].client_addr")"

			PORT=30000
			TMP="$(toml get "${CONFIGFILE}" driver | \
					jq -r ".[${INDEX}].client_port")"
			if [ ! "${TMP}" = "null" ]; then
				PORT="${TMP}"
			fi

			printf "%d. %s to client on %s on port %d, warehouses %d to %d.\n" \
					$(( INDEX + 1)) "${DRIVER_HOSTNAME}" "${CLIENT_HOSTNAME}" \
					"${PORT}" "${START}" "${END}"

			eval "${DRIVER_COMMAND} dbt2 driver -d ${CLIENT_HOSTNAME} \
					-p ${PORT} ${DRIVER_CMD_ARGS} > \
					${DRIVER_OUTPUT_DIR}/${HOSTNAME}/driver-${INDEX}.txt 2>&1" \
					|| exit 1 &
		fi

		# Calculate ramp up time for this driver and sleep only for the
		# duration of the driver with the longest ramp up time.

		OPTION1=$(( WMAX - WMIN + 1 ))
		OPTION1=$(( OPTION1 * THREADS_PER_WAREHOUSE ))
		THIS_RAMPUP_TIME=${OPTION1}
		if [ ! "${TERMINALS_LIMIT}" = "" ]; then
			OPTION2=${TERMINALS_LIMIT}
			if [ "${OPTION1}" -lt "${OPTION2}" ]; then
				THIS_RAMPUP_TIME=${OPTION1}
			else
				THIS_RAMPUP_TIME=${OPTION2}
			fi
		fi
		if [ "${DRIVER3}" = "1" ]; then
			THIS_RAMPUP_TIME=$(( ($(nproc) * FPP) - 1 ))
			if [ ! "${TERMINALS_LIMIT}" = "" ] &&
					[ "${TERMINALS_LIMIT}" -lt "${THIS_RAMPUP_TIME}" ]; then
				THIS_RAMPUP_TIME="${TERMINALS_LIMIT}"
			fi
		fi
		THIS_RAMPUP_TIME=$(( THIS_RAMPUP_TIME * THREAD_STARTUP_DELAY ))
		THIS_RAMPUP_TIME=$(( THIS_RAMPUP_TIME / 1000 ))

		if [ ${THIS_RAMPUP_TIME} -gt ${DRIVER_RAMPUP_TIME} ]; then
			DRIVER_RAMPUP_TIME=${THIS_RAMPUP_TIME}
		fi
	done

	DRIVERLIST="$(echo "${DRIVERLIST}" | \
			awk -v RS='[[:space:]]+' '!a[$0]++{printf "%s%s", $0, RT}')"
	echo
	printf "Driver systems:%s\n" "${DRIVERLIST}"

	DATABASES="$(toml get "${CONFIGFILE}" . | jq -r '.database | length')"
	for INDEX in $(seq 0 $(( DATABASES - 1 ))); do
		DB_HOSTNAME="$(toml get "${CONFIGFILE}" database | \
				jq -r ".[${INDEX}].database_addr")"
		DBLIST="${DBLIST} ${DB_HOSTNAME}"
	done
	DBLIST="$(echo "${DBLIST}" | \
			awk -v RS='[[:space:]]+' '!a[$0]++{printf "%s%s", $0, RT}')"

	echo
	printf "Database systems:%s\n" "${DBLIST}"
	echo
fi

# Start collecting system statistics while we ramp up the test.
stat_collection

printf "Waiting for driver(s) to ramp up, "
do_sleep $DRIVER_RAMPUP_TIME
echo

if [ "${PROFILE}" -eq 1 ]; then
	profile_collect &
fi

# Sleep for the duration of the run.
printf "Waiting for test to complete, "
do_sleep "${DURATION}"

# Stop stats collection.
stat_collection -s

echo
echo "## Stage 3. Results"
echo

if [ "${CONFIGFILE}" = "" ]; then
	# TODO: Make the tear down of the clients and drivers more graceful than
	# blowing away the client and waiting for drivers to stop complaining about
	# that it disappeared.

	if [ ! "$DRIVER3" = "1" ]; then
		# Client doesn't go away by itself like the driver does, so kill it.
		PIDFILE=$(eval "${CLIENT_COMMAND} find ${OUTPUT_DIR} \
				-name dbt2_client.pid" 2> /dev/null)
		CLIENTPID=$(eval "${CLIENT_COMMAND} cat ${PIDFILE}")
		eval "${CLIENT_COMMAND} kill -9 ${CLIENTPID}"
	fi

	# Make sure all the drivers have exited, so all the mix files have flushed
	# to disk before continuing processing.  Can't count on all of the sleeps
	# to be precise...
	eval "wait ${DRIVER_PIDLIST}"

	if [ $PRIVILEGED -eq 1 ]; then
		eval "${DB_COMMAND} dbt2 ${DBMS}-stop-db ${MISC_ARG}"
	fi

	if [ "${PROFILE}" -eq 1 ]; then
		profile_process
	fi

	# Retrieve test data from all remote systems.
	if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		# If a database system is specified, rsync all the logs back to the
		# driver system after creating the sar csv data.  There will only be a
		# sar output file if the database is on a separate system.
		if [ $DBAAS -eq 0 ]; then
			rsync -a -e "$SSH" --delete "${DB_HOSTNAME}:${DB_OUTPUT_DIR}/" \
					"${DB_OUTPUT_DIR}/"
		fi
	fi

	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		rsync -a -e "${SSH}" --delete \
				"${CLIENT_HOSTNAME}:${CLIENT_OUTPUT_DIR}/" \
				"${CLIENT_OUTPUT_DIR}/"
	fi
else
	INDEX=0
	for SYSTEM in ${DRIVERLIST}; do
		if [ "${SYSTEM}" = "localhost" ]; then
			CMD=""
		else
			CMD="${SSH} ${SYSTEM}"
		fi
	done

	eval "${CMD} find ${OUTPUT_DIR}/driver -name dbt2_driver.pid -print" | \
			while IFS= read -r PIDFILE; do
		PID=$(eval "${CMD} cat ${PIDFILE}")
		INDEX=$(( INDEX + 1 ))
		printf "${INDEX}. Waiting for driver (PID %d) to stop on %s." \
				"${PID}" "${SYSTEM}"
		while eval "${CMD} kill -0 ${PID}" 2> /dev/null; do
			printf "."
			sleep 1
		done
		echo
	done
	echo

	if [ ! "$DRIVER3" = "1" ]; then
		for SYSTEM in ${CLIENTLIST}; do
			if [ "${SYSTEM}" = "localhost" ]; then
				CMD=""
			else
				CMD="${SSH} ${SYSTEM}"
			fi
			for PIDFILE in $(eval "${CMD} find ${OUTPUT_DIR} \
					-name dbt2_client.pid" 2> /dev/null); do
				CLIENTPID=$(eval "${CMD} cat ${PIDFILE}")
				eval "${CMD} kill -9 ${CLIENTPID}"
			done
		done
	fi

	# There shouldn't be any files on the database system to gather unless
	# system stats or software profiles are collected.
	if [ ${STATS} -eq 1 ] || [ ${PROFILE} -eq 1 ]; then
		SYSTEMLIST="$(echo "${DRIVERLIST} ${CLIENTLIST} ${DBLIST}" | \
				awk -v RS='[[:space:]]+' '!a[$0]++{printf "%s%s", $0, RT}')"
	else
		SYSTEMLIST="$(echo "${DRIVERLIST} ${CLIENTLIST}" | \
				awk -v RS='[[:space:]]+' '!a[$0]++{printf "%s%s", $0, RT}')"
	fi
	for SYSTEM in ${SYSTEMLIST}; do
		if [ ! "${SYSTEM}" = "localhost" ]; then
			rsync -a -e "${SSH}" "${SYSTEM}:${OUTPUT_DIR}/" "${OUTPUT_DIR}/"
		fi
	done
fi

dbt2 report "${OUTPUT_DIR}" > "${SUMMARY}"
echo "Results are in: ${OUTPUT_DIR}"
echo

cat "${SUMMARY}"
