#!/bin/sh

#
# This file is released under the terms of the Artistic License.
# Please see the file LICENSE, included in this package, for details.
#
# Copyright The DBT-2 Authors
#

do_sleep()
{
	echo "Sleeping $1 seconds"
	sleep "${1}"
}

make_directories()
{
	COMMAND=""
	HOST=${1}
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	fi
	${COMMAND} mkdir -p "${OUTPUT_DIR}"
}

profile_collect()
{
	if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		which ts ts-profile > /dev/null 2>&1
		RC=$?
		if [ $RC -gt 1 ]; then
			echo "WARNING: touchstone tools not found for local profiling"
		else
			ts profile -o "${DRIVER_OUTPUT_DIR}/$(hostname)/profile" sleep 10 &
		fi
	fi

	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		eval "${CLIENT_COMMAND} \"which ts ts-profile > /dev/null 2>&1\""
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for profiling %s\n" \
					"${HOSTNAME}"
		else
			eval "${CLIENT_COMMAND} ts profile \
					-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}/profile sleep 10 &"
		fi
	fi

	if [ $DBAAS -eq 0 ]; then
		HOSTNAME=$(eval "${DB_COMMAND} hostname")
		eval "${DB_COMMAND} \"which ts ts-profile > /dev/null 2>&1\""
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for profiling %s\n" \
					"${HOSTNAME}"
		else
			eval "${DB_COMMAND} ts profile \
					-o ${DB_OUTPUT_DIR}/${HOSTNAME}/profile sleep 10 &"
		fi
	fi
}

profile_process()
{
	echo "processing profiling data..."
	if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		ts profile -o "${DRIVER_OUTPUT_DIR}/$(hostname)/profile" -p
	fi

	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		eval "${CLIENT_COMMAND} ts profile \
				-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}/profile -p"
	fi

	if [ $DBAAS -eq 0 ]; then
		HOSTNAME=$(eval "${DB_COMMAND} hostname")
		eval "${DB_COMMAND} ts profile \
				-o ${DB_OUTPUT_DIR}/${HOSTNAME}/profile -p"
	fi
}

stat_collection()
{
	if [ $STATS -ne 1 ]; then
		return
	fi

	ARGS=$1

	# If the this is a 1-tier test, system stats will be saved only in the db
	# directory.
	if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
		which ts ts-profile > /dev/null 2>&1
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for driver system "
			echo "stats collection"
		else
			eval "ts sysstat -o ${DRIVER_OUTPUT_DIR}/$(hostname)/sysstat ${ARGS} &"
		fi
	fi

	# Handle system statistics on the Client node, if remote.
	if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
		HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
		eval "${CLIENT_COMMAND} \"which ts ts-profile > /dev/null 2>&1\""
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for local system "
			echo "stats collection"
		else
			eval "${CLIENT_COMMAND} ts sysstat \
					-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}/sysstat ${ARGS} &"
		fi
	fi

	# Don't collect system stats from the database system if the database
	# is a service.
	HOSTNAME=$(eval "${DB_COMMAND} hostname")
	if [ $DBAAS -eq 0 ]; then
		eval "${DB_COMMAND} which ts ts-sysstat > /dev/null 2>&1"
		RC=$?
		if [ $RC -gt 1 ]; then
			printf "WARNING: touchstone tools not found for system stats "
			printf "collection on %s\n" "${HOSTNAME}"
		else
			eval "${DB_COMMAND} ts sysstat \
					-o ${DB_OUTPUT_DIR}/${HOSTNAME}/sysstat ${ARGS} &"
		fi
	fi

	eval "${DB_COMMAND} which ts ts-${DBMS}-stat" > /dev/null 2>&1
	RC=$?
	if [ $RC -gt 1 ]; then
		printf "WARNING: touchstone tools not found for database stats "
		printf "collection for RDBMS %s on system %s\n" "${DBMS}" "${HOSTNAME}"
	else
		eval "${DB_COMMAND} ts ${DBMS}-stat \
				-o ${DB_OUTPUT_DIR}/${HOSTNAME}/dbstat ${DBSTATARGS} ${ARGS} &"
	fi
}

stop_processes() {
	if [ "$DRIVER3" = "1" ]; then
		killall dbt2-driver3
	else
		killall dbt2-driver
		if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
			eval "${CLIENT_COMMAND} killall dbt2-client"
		fi
	fi

	"dbt2 ${DBMS}-stop-db"

	# Stop stat collection.
	stat_collection -s
}

trap 'echo "Test was interrupted by Control-C."; stop_processes' INT
trap 'echo "Test was interrupted. Got TERM signal."; stop_processes' TERM

usage()
{
	if [ "$1" != "" ]; then
		echo
		echo "error: $1"
		echo
	fi
	cat << EOF
$(basename "${0}") is the Database Test 2 (DBT-2) workload runner

Usage:
  $(basename "${0}") [OPTIONS] DBMS DIRECTORY

General options:
  -A, --advanced
                 enable advanced benchmarking execution options
  --comment=COMMENTS
                 provide COMMENTS to save with the test
  --connection-delay=MILLISECONDS
                 MILLISECONDS delay between client and database connections,
                 default 100
  -d SECONDS, --duration=SECONDS
                 SECONDS of the test duration, after ramp up, default $DURATION
  --db-host=ADDRESS
                 ADDRESS of database system, default localhost
  --db-name=NAME
                 NAME of the database, default $DBNAME
  --db-port=PORT
                 database listening PORT number
  --db-user=USER
                 database USER
  --dbaas        database server is a service
  --districts=DISTRICTS
                 number of DISTRICTS per warehouse, default 10
  --driver-mode=MODE
                 1 for threaded, 3 for event-driven process MODE, default 3
  --profile      enable software profiling collection
  --privileged   run test as a privileged operating system and database user
  --stats        collect system and database stats
  -V, --version  output version information, then exit
  -w WAREHOUSES, --warehouses=WAREHOUSES
                 number of WAREHOUSES to use in the database, default $WAREHOUSES
  -?, --help     show this help, then exit

driver-mode 1 (threaded) specific options:
  -c CONNECTIONS, --client-connections=CONNECTIONS
                 number of database CONNECTIONS to open, default $DBCON
  --client-host=ADDRESS
                 ADDRESS of client system, default localhost
  --driver-partition-size=NUMBER
                 NUMBER of warehouses per process, default all
  --terminal-limit=TERMINALS
                 limit the number of TERMINALS emulated

driver-mode 3 (event-driven, process based) specific options:
  --connections-per-processor=PROCESSES
                 driver PROCESSES started per processor, default 1
  --terminal-limit=CONNECTIONS
                 limit the number of database CONNECTIONS opened

MySQL specific options:
  --db-password=PASSWORD
                 database PASSWORD

PostgreSQL specific options:
  --db-parameters=OPTIONS
                 GUC command line OPTIONS to pass to postgres, when running
                 by privileged user

DBMS options are:
  cockroach  CockroachDB
  mysql      MySQL
  pgsql      PostgreSQL
  sqlite     SQLite

DIRECTORY is the path to save test results.

Database Test 2 (DBT-2) project page: https://github.com/osdldbt/dbt2
EOF
}

validate_parameter()
{
	if [ "$2" != "$3" ]; then
		usage "wrong argument '$2' for parameter '-$1'"
		exit 1
	fi
}

# Tell ssh not to check known hosts or host keys.
SSH="ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

ALTERED_ARG=""
CLIENT_HOSTNAME="localhost"
DB_HOSTNAME="localhost"
DB_PASSWORD=""
DBAAS=0
DBCON=10
DBNAME="dbt2"
DRIVER3=1
DURATION=180
EASYFLAGS="-ktd 0 -ktn 0 -kto 0 -ktp 0 -kts 0 -ttd 0 -ttn 0 -tto 0 -ttp 0 -tts 0"
FPP=1
PRIVILEGED=0
PROFILE=0
STATS=0
THREAD_STARTUP_DELAY=100 # milliseconds
THREADS_PER_WAREHOUSE=10
WAREHOUSES=1

# Custom argument handling for hopefully most portability.
while [ "${#}" -gt 0 ] ; do
	opt=${1}
	case "${1}" in
	(-A | --advanced)
		# Advanced benchmarking usage.  Disable all "easy" flags.
		EASYFLAGS=""
		;;
	(-c | --client-connections)
		shift
		# Check for numeric value
		DBCON=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DBCON}"
		;;
	(-c?*)
		# Check for numeric value
		DBCON=$(echo "${1#*-c}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DBCON}"
		;;
	(--client-connections=?*)
		# Check for numeric value
		DBCON=$(echo "${1#*--client-connections=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DBCON}"
		;;
	(--client-host)
		shift
		CLIENT_HOSTNAME=$1
		;;
	(--client-host=?*)
		CLIENT_HOSTNAME=${1#*--client-host=}
		;;
	(--comment)
		shift
		COMMENT=$1
		;;
	(--comment=?*)
		COMMENT=${1#*--comment=}
		;;
	(--connection-delay)
		shift
		THREAD_STARTUP_DELAY=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${THREAD_STARTUP_DELAY}"
		;;
	(--connection-delay=?*)
		THREAD_STARTUP_DELAY=$(echo "${1#*--connection-delay=}" \
				| grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--connection-delay=}" \
				"${THREAD_STARTUP_DELAY}"
		;;
	(--connections-per-processor)
		shift
		FPP="$1"
		FPP_FLAG="-fpp $FPP"
		;;
	(--connections-per-processor=?*)
		FPP="${1#*--connections-per-processor=}"
		FPP_FLAG="-fpp $FPP"
		;;
	(-d | --duration)
		shift
		DURATION=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DURATION}"
		;;
	(-d?*)
		DURATION=$(echo "${1#*-d}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*-d}" "${DURATION}"
		;;
	(--duration=?*)
		DURATION=$(echo "${1#*--duration=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--duration=}" "${DURATION}"
		;;
	(--db-host)
		shift
		DB_HOSTNAME=${1}
		;;
	(--db-host=?*)
		DB_HOSTNAME=${1#*--db-host=}
		;;
	(--db-name)
		shift
		DBNAME=${1}
		;;
	(--db-name=?*)
		DBNAME=${1#*--db-name=}
		;;
	(--db-parameters)
		shift
		DB_PARAMS="${1}"
		DB_PARAMS_ARG="-p \"${DB_PARAMS}\""
		;;
	(--db-parameters=?*)
		DB_PARAMS="${1#*--db-parameters=}"
		DB_PARAMS_ARG="-p \"${DB_PARAMS}\""
		;;
	(--db-password)
		shift
		DB_PASSWORD=${1}
		;;
	(--db-password=?*)
		DB_PASSWORD=${1#*--db-password=}
		;;
	(--db-port)
		shift
		DB_PORT=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${DB_PORT}"
		;;
	(--db-port=?*)
		DB_PORT=$(echo "${1#*--db-port=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--db-port=}" "${DB_PORT}"
		;;
	(--db-user)
		shift
		DB_USER=${1}
		;;
	(--db-user=?*)
		DB_USER=${1#*--db-user=}
		;;
	(--dbaas)
		DBAAS=1
		;;
	(--districts)
		shift
		THREADS_PER_WAREHOUSE=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${THREADS_PER_WAREHOUSE}"
		;;
	(--districts=?*)
		THREADS_PER_WAREHOUSE=$(echo "${1#*--districts=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--districts=}" \
				"${THREADS_PER_WAREHOUSE}"
		;;
	(--driver-mode)
		shift
		# Logic is pretty basic at the moment, set to 3 or unset it.
		if [ "${1}" -ne 3 ]; then
			DRIVER3=""
		fi
		;;
	(--driver-mode=?*)
		# Logic is pretty basic at the moment, set to 3 or unset it.
		if [ "${1#*--driver-mode=}" -ne 3 ]; then
			DRIVER3=""
		fi
		;;
	(--driver-partition-size)
		shift
		# Number of warehouses per driver.
		W_CHUNK=${1}
		;;
	(--driver-partition-size=?*)
		# Number of warehouses per driver.
		W_CHUNK=${1#*--driver-partition-size=}
		;;
	(--privileged)
		PRIVILEGED=1
		;;
	(--profile)
		PROFILE=1
		;;
	(-S)
		shift
		SOCK_FILE_ARG="-t ${1}"
		;;
	(-S?*)
		SOCK_FILE_ARG="-t ${1#*-S}"
		;;
	(--stats)
		STATS=1
		;;
	(--terminal-limit)
		shift
		TERMINALS_LIMIT="$1"
		TERMINALS_LIMIT_ARG="-L ${TERMINALS_LIMIT}"
		;;
	(--terminal-limit=?*)
		TERMINALS_LIMIT="${1#*--terminal-limit=}"
		TERMINALS_LIMIT_ARG="-L ${TERMINALS_LIMIT}"
		;;
	(-V | --version)
		echo "$(basename "${0}") v0.53.9"
		exit 0
		;;
	(-w | --warehouses)
		shift
		WAREHOUSES=$(echo "${1}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1}" "${WAREHOUSES}"
		;;
	(-w?*)
		WAREHOUSES=$(echo "${1#*-w}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*-w}" "${WAREHOUSES}"
		;;
	(--warehouses=?*)
		WAREHOUSES=$(echo "${1#*--warehouses=}" | grep -E "^[0-9]+$")
		validate_parameter "${opt}" "${1#*--warehouses=}" "${WAREHOUSES}"
		;;
	(-\? | --help)
		usage
		exit 0
		;;
	(--* | -*)
		echo "$(basename "${0}"): invalid option -- '${1}'"
		echo "try \"$(basename "${0}") --help\" for more information."
		exit 1
		;;
	(*)
		break
		;;
	esac
	shift
done

if [ $# -eq 0 ]; then
	printf "Specify which DBMS system to test, try \"%s --help\" " \
			"$(basename "${0}")"
	echo "for more information."
	exit 1
fi

DBMS=$1
shift

if [ $# -eq 0 ]; then
	echo "Specify the location to save test results, where directory does "
	printf "not exist yet, try \"%s\" " "$(basename "${0}")"
	echo "for more information."
	exit 1
fi

OUTPUT_DIR=$1

# If no chunking of warehouses per process is defined, assumed we want to run
# everything from one process.
if [ "${W_CHUNK}" = "" ]; then
	W_CHUNK=${WAREHOUSES}
fi

# Check parameters.

if [ -d "${OUTPUT_DIR}" ]; then
	echo "Directory '${OUTPUT_DIR}' exists, stopping to prevent possible"
	echo "clobbering of data, please specify another location."
	exit 1
fi

if [ "${THREADS_PER_WAREHOUSE}" -lt 1 ] || \
		[ "${THREADS_PER_WAREHOUSE}" -gt 1000 ];
then
	usage "--districts value should be in range [1..1000]. Please specify correct value"
	exit 1
fi

if [ ! "${DB_PORT}" = "" ]; then
	if [ "$DRIVER3" = "1" ]; then
		DB_PORT_ARG="-P ${DB_PORT}"
	else
		DB_PORT_ARG="-l ${DB_PORT}"
	fi
fi

# Enable altered execution mode only if DRIVER3 is enabled.
if [ "${DRIVER3}" = "1" ]; then
	ALTERED_ARG="-altered ${DRIVER3}"
fi

DBSTATARGS=""
case $DBMS in
pgsql)
	# Set extra argument variables.
	DBSTATARGS="-h $DB_HOSTNAME -d $DBNAME"
	if [ ! "${DB_USER}" = "" ]; then
		DBSTATARGS="${DBSTATARGS} -U ${DB_USER}"
	fi
	if [ ! "${DB_PORT}" = "" ]; then
		DBSTATARGS="${DBSTATARGS} -p ${DB_PORT}"
	fi
	;;
esac

CLIENT_OUTPUT_DIR=${OUTPUT_DIR}/client
DRIVER_OUTPUT_DIR=${OUTPUT_DIR}/driver
DB_OUTPUT_DIR=${OUTPUT_DIR}/db

SUMMARY="${OUTPUT_DIR}/summary.rst"

# Check if rsync exists on all systems and stop if not, required for collecting
# data from all systems.
if ! which rsync > /dev/null 2>&1; then
	echo "ERROR: rsync required on current system in order to retrieve results"
	exit 1
fi

if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
	CLIENT_COMMAND="$SSH $CLIENT_HOSTNAME"
	# Check for rsync on the client before continuing.
	if ! eval "${CLIENT_COMMAND} which rsync > /dev/null 2>&1"; then
		printf "ERROR: rsync required on client system in order to retrieve "
		echo "results"
		exit 1
	fi
else
	CLIENT_COMMAND=""
fi

if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ] && \
		[ $DBAAS -eq 0 ]; then
	DB_COMMAND="$SSH ${DB_HOSTNAME}"
	# Check for rsync on the client before continuing.
	if ! eval "${DB_COMMAND} which rsync > /dev/null 2>&1"; then
		printf "ERROR: rsync required on database system in order to retrieve "
		echo "results"
		exit 1
	fi
else
	DB_COMMAND=""
fi

#
# Create the directories we will need.
#
make_directories
mkdir -p "${DRIVER_OUTPUT_DIR}/$(hostname)"
if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
	make_directories "${CLIENT_HOSTNAME}"
	HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
	eval "${CLIENT_COMMAND} mkdir -p ${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
fi
if [ ! "${DB_HOSTNAME}" = "localhost" ]; then
	#
	# Create direcotires on the database server if remote.
	#
	if [ $DBAAS -eq 0 ]; then
		make_directories "${DB_HOSTNAME}"
		$SSH "$DB_HOSTNAME" -- mkdir -p "$DB_OUTPUT_DIR"
	fi
fi
mkdir -p "${DB_OUTPUT_DIR}"

# Create a readme file in the output directory, date it and add run information
# into the readme.txt.
cat >> "${OUTPUT_DIR}/readme.txt" << EOF
$(date)
$COMMENT
Command line: $0 $*
RDBMS: $DBMS
Database Name: $DBNAME
Database Scale Factor: $WAREHOUSES warehouses
Test Duration: $DURATION seconds
Database Connections: $DBCON
EOF

# Get any OS specific information, if this is a single tier configuration, then
# the OS parameters will be in the database output directory.
if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
	dbt2 get-os-info -o "${DRIVER_OUTPUT_DIR}/$(hostname)"
	HOSTNAME=$(eval "${DB_COMMAND} hostname")
	eval "${DB_COMMAND} dbt2 get-os-info -o ${DB_OUTPUT_DIR}/${HOSTNAME}"
else
	dbt2 get-os-info -o "${DB_OUTPUT_DIR}/$(hostname)"
fi

if [ $PRIVILEGED -eq 1 ]; then
	if eval "${DB_COMMAND} dbt2 ${DBMS}-stop-db ${MISC_ARG}"; then
		echo "cleaning up results..."
		rm -rf "${OUTPUT_DIR}"
		exit 1
	fi
	eval "${DB_COMMAND} dbt2 ${DBMS}-start-db ${DB_PARAMS_ARG} ${MISC_ARG} \
			-o ${DB_OUTPUT_DIR}" || exit 1
fi

#
# Redisplay the test parameters.
#
echo "DBT-2 test for ${DBMS} started at $(date)"
echo ""
echo "DATABASE SYSTEM: ${DB_HOSTNAME}"
if [ "${DBMS}" = "cockroach" ] || [ "${DBMS}" = "pgsql" ] || \
		[ "${DBMS}" = "mysql" ]; then
	echo "DATABASE NAME: ${DBNAME}"
fi

#
# Build up the client command line arguments.
#

CLIENT_COMMAND_ARGS="-a $DBMS"

if [ -n "${DB_USER}" ]; then
	echo "DATABASE USER: ${DB_USER}"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -u ${DB_USER}"
fi

if [ -n "${DB_PASSWORD}" ]; then
	echo "DATABASE PASSWORD: *******"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -a ${DB_PASSWORD}"
fi

if [ -n "${DB_SOCKET}" ]; then
	echo "DATABASE SOCKET: ${DB_SOCKET}"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -t ${DB_SOCKET}"
fi

if [ -n "${DB_PORT}" ]; then
	echo "DATABASE PORT: ${DB_PORT}"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} ${DB_PORT_ARG}"
fi

OPTION1=$(( WAREHOUSES * THREADS_PER_WAREHOUSE ))
THREADS=$OPTION1
if [ ! "$TERMINALS_LIMIT" = "" ]; then
	OPTION2=$TERMINALS_LIMIT
	if [ "${OPTION1}" -lt "${OPTION2}" ]; then
		THREADS=$OPTION1
	else
		THREADS=$OPTION2
	fi
fi
if [ "$DRIVER3" = "1" ]; then
	echo "DATABASE CONNECTIONS: $(nproc)"
	echo "TERMINAL PROCESSES: $(nproc)"
else
	echo "DATABASE CONNECTIONS: ${DBCON}"
	echo "TERMINAL THREADS: ${THREADS}"
fi
echo "TERMINALS PER WAREHOUSE: ${THREADS_PER_WAREHOUSE}"
if [ ! "$DRIVER3" = "1" ]; then
	echo "TERMINAL CAP: $TERMINALS_LIMIT"
fi
echo "WAREHOUSES PER DRIVER/CLIENT PAIR: $W_CHUNK"
echo "SCALE FACTOR (WAREHOUSES): ${WAREHOUSES}"
echo "DURATION OF TEST (in sec): ${DURATION}"
echo ""

#
# Start the client.
#
echo "Stage 1. Starting up client..."
if [ ! "$DRIVER3" = "1" ]; then
	printf "database connection starting every %s" "${THREAD_STARTUP_DELAY}"
	echo "milliseconds"
	if [ "${DBMS}" = "cockroach" ]; then
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_HOSTNAME}"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -b ${DBNAME}"
	elif [ "${DBMS}" = "mysql" ]; then
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DBNAME}"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} ${SOCK_FILE_ARG} -u root"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -h ${DB_HOSTNAME}"
	elif [ "${DBMS}" = "pgsql" ]; then
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_HOSTNAME}"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -b ${DBNAME}"
	elif [ "${DBMS}" = "sqlite" ]; then
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_PARAMS}"
	fi
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -f -c $DBCON"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -s ${THREAD_STARTUP_DELAY}"
	END=0
	START=1

	#
	# Count of the number of clients to run.
	#
	SEG=0
	PORT=30000
	while [ "${END}" -lt "${WAREHOUSES}" ]; do
		START=$(( END + 1 ))
		END=$(( START + W_CHUNK ))
		END=$(( END - 1 ))
		SEG=$(( SEG + 1 ))

		if [ "${END}" -gt "${WAREHOUSES}" ]; then
			END=${WAREHOUSES}
		fi

		HOSTNAME=$($CLIENT_COMMAND hostname)
		CPDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
		mkdir -p "${CPDIR}"

		CDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}.${SEG}"
		eval "${CLIENT_COMMAND} mkdir -p ${CDIR}"

		CARGS="${CLIENT_COMMAND_ARGS} -p ${PORT} -o ${CDIR}"
		if [ ! "$DRIVER3" = "1" ]; then
			eval "${CLIENT_COMMAND} dbt2 client ${CARGS} \
					> ${CPDIR}/client-${SEG}.txt 2>&1 &"
		fi

		PORT=$(( PORT + 1))
	done

	# Sleep long enough for all the client database connections to be
	# established.
	CLIENT_RAMPUP_TIME=$DBCON
	CLIENT_RAMPUP_TIME=$(( CLIENT_RAMPUP_TIME * THREAD_STARTUP_DELAY ))
	CLIENT_RAMPUP_TIME=$(( CLIENT_RAMPUP_TIME / 1000 ))
	CLIENT_RAMPUP_TIME=$(( CLIENT_RAMPUP_TIME + 1 ))
	do_sleep $CLIENT_RAMPUP_TIME
else
	echo "***skipping***"
fi

# Calculate how many terminals are being emulated.
OPTION1=$(( W_CHUNK + 1 ))
OPTION1=$(( OPTION1 * THREADS_PER_WAREHOUSE ))
DRIVER_RAMPUP_TIME=$OPTION1
if [ ! "$TERMINALS_LIMIT" = "" ]; then
	OPTION2=$TERMINALS_LIMIT
	if [ "${OPTION1}" -lt "${OPTION2}" ]; then
		DRIVER_RAMPUP_TIME=$OPTION1
	else
		DRIVER_RAMPUP_TIME=$OPTION2
	fi
fi
if [ "$DRIVER3" = "1" ]; then
	DRIVER_RAMPUP_TIME=$(( ($(nproc) * FPP) - 1 ))
	if [ ! "$TERMINALS_LIMIT" = "" ] &&
			[ $TERMINALS_LIMIT -lt $DRIVER_RAMPUP_TIME ]; then
		DRIVER_RAMPUP_TIME="${TERMINALS_LIMIT}"
	fi
fi
DRIVER_RAMPUP_TIME=$(( DRIVER_RAMPUP_TIME * THREAD_STARTUP_DELAY ))
DRIVER_RAMPUP_TIME=$(( DRIVER_RAMPUP_TIME / 1000 ))
TOTAL_RUN_TIME=$(( DRIVER_RAMPUP_TIME + DURATION ))
SAMPLE_LENGTH=60
ITERATIONS=$(( TOTAL_RUN_TIME / SAMPLE_LENGTH ))
ITERATIONS=$(( ITERATIONS + 1 ))

# Start collecting data before we start the test.
stat_collection

if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
	HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
	eval "${CLIENT_COMMAND} dbt2 get-os-info \
			-o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
fi

echo "collecting database statistics..."
HOSTNAME=$(eval "${DB_COMMAND} hostname")
eval "$DB_COMMAND mkdir -p ${DB_OUTPUT_DIR}/${HOSTNAME}"
if [ $DBAAS -eq 0 ]; then
	eval "${DB_COMMAND} dbt2 ${DBMS}-plans -d ${DBNAME} \
			-o ${DB_OUTPUT_DIR}/${HOSTNAME}/plan0.txt"
fi

# Start the driver.
echo ''
echo "Stage 2. Starting up driver..."
echo "connection opening every $THREAD_STARTUP_DELAY milliseconds"

END=0
START=1
#
# Count of the number of drivers run.
#
SEG=0
PORT=30000
while [ "${END}" -lt "${WAREHOUSES}" ]; do
	START=$(( END + 1 ))
	END=$(( START + W_CHUNK ))
	END=$(( END - 1 ))
	SEG=$(( SEG + 1 ))

	if [ "${END}" -gt "${WAREHOUSES}" ]; then
		END=${WAREHOUSES}
	fi

	DDIR="${DRIVER_OUTPUT_DIR}/$(hostname).${SEG}"
	mkdir -p "${DDIR}"
	DRIVER_COMMAND_ARGS="-l $DURATION -wmin $START"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -wmax $END -w $WAREHOUSES"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -sleep $THREAD_STARTUP_DELAY"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -outdir $DDIR"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -tpw $THREADS_PER_WAREHOUSE"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS $EASYFLAGS $TERMINALS_LIMIT_ARG"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS $ALTERED_ARG $DB_PORT_ARG"
	if [ "$DRIVER3" = "1" ]; then
		DHNARG="-b $DBNAME -d $DB_HOSTNAME"
		eval "dbt2 driver3 $DRIVER_COMMAND_ARGS -a $DBMS $DHNARG $FPP_FLAG \
				> ${DRIVER_OUTPUT_DIR}/$(hostname)/driver-${SEG}.txt 2>&1" \
				|| exit 1 &
	else
		eval "dbt2 driver -d $CLIENT_HOSTNAME -p $PORT $DRIVER_COMMAND_ARGS > \
				${DRIVER_OUTPUT_DIR}/$(hostname)/driver-${SEG}.txt 2>&1" \
				|| exit 1 &
	fi
	DRIVER_PIDLIST="$DRIVER_PIDLIST $!"
	PORT=$(( PORT + 1 ))
done

printf "estimated rampup time: "
do_sleep $DRIVER_RAMPUP_TIME
echo "estimated rampup time has elapsed"

if [ "${PROFILE}" -eq 1 ]; then
	profile_collect &
fi

# Sleep for the duration of the run.
printf "estimated steady state time: "
do_sleep "${DURATION}"

# Stop stats collection.
stat_collection -s

echo ''
echo "Stage 3. Processing of results..."

# TODO: Make the tear down of the clients and drivers more graceful than
# blowing away the client and waiting for drivers to stop complaining about
# that it disappeared.

if [ ! "$DRIVER3" = "1" ]; then
	# Client doesn't go away by itself like the driver does, so kill it.
	echo "Killing client..."
	eval "${CLIENT_COMMAND} killall -q dbt2-client > /dev/null 2>&1 &"
fi

# Make sure all the drivers have exited, so all the mix files have flushed to
# disk before continuing processing.  Can't count on all of the sleeps to be
# precise...
eval "wait ${DRIVER_PIDLIST}"

dbt2 report "${OUTPUT_DIR}" > "${SUMMARY}"

if [ $PRIVILEGED -eq 1 ]; then
	eval "${DB_COMMAND} dbt2 ${DBMS}-stop-db ${MISC_ARG}"
fi

if [ "${PROFILE}" -eq 1 ]; then
	profile_process
fi

# Retrieve test data from all remote systems.
if [ -n "${DB_HOSTNAME}" ] && [ ! "${DB_HOSTNAME}" = "localhost" ]; then
	# If a database system is specified, rsync all the logs back to the driver
	# system after creating the sar csv data.  There will only be a sar output
	# file if the database is on a separate system.
	if [ $DBAAS -eq 0 ]; then
		rsync -a -e "$SSH" --delete "${DB_HOSTNAME}:${DB_OUTPUT_DIR}/" \
				"${DB_OUTPUT_DIR}/"
	fi
fi

if [ ! "${CLIENT_HOSTNAME}" = "localhost" ]; then
	HOSTNAME=$(eval "${CLIENT_COMMAND} hostname")
	rsync -a -e "${SSH}" --delete "${CLIENT_HOSTNAME}:${CLIENT_OUTPUT_DIR}/" \
			"${CLIENT_OUTPUT_DIR}/"
fi

echo "Test completed."
echo "Results are in: ${OUTPUT_DIR}"
echo

cat "${SUMMARY}"
