#!/bin/bash
# bsr -> buffered sequential reads
# brr -> Buffered random reads
# brrmmap -> buffered random read on mmapped files
# bufw -> Buffered Writes
# bufwfs -> Buffered Writes with fsync issued at the end of job 
# bufwfs32 -> Buffered Writes with fsync issued after every 32 requests
# osyncw -> O_SYNC writes
# drr -> direct random reads
# drw -> direct random writes
# drrmmap -> direct random read on mmapped files
# arr -> asynchronous random reads
# arw -> asynchronous random writes
# mixed --> mixed, direct seq read, direct rand read, buffered write
# database--> primarily AIO + O_DIRECT
# desktop--> buffered sequential and random reads + buffered writes
# kvmhost--> AIO + O_DIRECT reads and writes + sequential reads and writes

# default mount point
MNTPOINT=/mnt/iostestmnt
CGROUPMNTPOINT=/cgroup/blkio
CGBASEWEIGHT=100
CGBASESTRING="cgrp"
# Max number of groups allowed is 8
MAXNRGRP=8
TESTDIR=$MNTPOINT/fio
OUTPUTDIR="`pwd`/iostest-results/"

# declare that array is associative. This feature is supported in bash 4.0. So
# iostest will not work with old bash.
declare -A WORKLOAD_MAP_ARRAY
WORKLOAD_MAP_ARRAY=(
		[bsr]="bsr"
		[brr]="brr"
		[brrmmap]="brrmmap"
		[bufw]="bufw"
		[bufwfs]="bufwfs"
		[bufwfs32]="bufwfs32"
		[osyncw]="osyncw"
		[drr]="drr"
		[drw]="drw"
		[drrmmap]="drrmmap"
		[arr]="arr"
		[arw]="arw"
		[mixed]="bsr brr bufw"
		[database]="arr arw"
		[desktop]="bsr bufw brr"
		[kvmhost]="arr arw bsr bufw brr"
	)

declare -A WORKLOAD_STRING_ARRAY
WORKLOAD_STRING_ARRAY=(
		[bsr]="Buffered Sequential Reads"
		[brr]="Buffered Random Reads"
		[brrmmap]="Buffered random reads on mmaped files"
		[bufw]="Buffered writes"
		[bufwfs]="Buffered writes with fsync at the end"
		[bufwfs32]="Buffered writes with fsync at every 32 IO"
		[osyncw]="O_SYNC Writes"
		[drr]="O_DIRECT Random Reads"
		[drw]="O_DIRECT Random Writes"
		[drrmmap]="O_DIRECT random reads on mmaped files"
		[arr]="AIO + O_Direct Random Reads"
		[arw]="AIO + O_Direct Random Writes"
		[mixed]="Mix of seq read, random read, buffered writes"
		[database]="AIO + O_DIRECT reads and writes"
		[desktop]="Buffered [seq and rnd ] reads and writes"
		[kvmhost]="AIO + O_DIRECT r+w + Buffered r+w"
	)

# Define an associateve array for group data. For each group defined on
# the command line using option "-g" we will create an entry here. The
# format of entry will look like as follows.
#
# index-> array will be indexed with group name.
# data--> data of each entry will be a string. This string will contain
#         many semicolon separated fields.
#  [test1] ="<data1> <data2> <data3>...."
# data1 --> weight of the group
# data2 --> Workload to be run in the group. This string will represent either
#	    single workload or top level composite wl. (bsr, desktop, custom)
# Field separator: #
#
# This array is created dynamically based on user inputs.

declare -A GROUP_DATA_ARRAY

# Print all the component jobs of a workload. This is static mapping.
component_jobs_of_workload () {
	local workload=$1

	echo ${WORKLOAD_MAP_ARRAY[$workload]}
}

# fio --minimal output varies with fio versions. Change parse method based on
# version info
set_fio_result_parse_method () {
	local fioversion=$1

	# For fio versions less than 1.41.4, we have legacy format.
	if [[ "$fioversion" < "1.41.4" ]] || [ "$fioversion" == "" ];then
		FIOPARSEMETHOD=1
	elif [ "$fioversion" == "1.41.4" ];then
		# Total latency stats have been addded but no additional
		# versioning field
		FIOPARSEMETHOD=2
	elif [[ "$fioversion" > "1.41.4" ]];then
		# Total latency stats have been addded and also contains
		# version number in first field.
		FIOPARSEMETHOD=3
	fi
}

#Helper functions
start_blktrace () {
	if [ -z "$BLKTRACE" ];then
		return
	fi

	# Currently in FSMODE, blktrace is not supported if user has not
	# specified a device.

	[ -n "$FSMODE" ] && [ -z "$BLKTRACEDEV" ] && return

	# mount debugfs
	mount -t debugfs none /sys/kernel/debug > /dev/null 2>&1
	
	# Direct blktrace output to result dir
	blktrace -d $BLKTRACEDEV -D $OUTPUTDIR/ &
	PIDBLKTRACE=$!
}

stop_blktrace () {
	if [ -z "$BLKTRACE" ];then
		return
	fi

	[ -n "$FSMODE" ] && [ -z "$BLKTRACEDEV" ] && return

	if [ -n "$PIDBLKTRACE" ];then
		kill -9 $PIDBLKTRACE > /dev/null 2>&1
		sleep 4
	fi

	blktrace -d $BLKTRACEDEV -k
}

cleanup_before_exit () {

	[ -n "$FSMODE" ] && return

	# unmount
	umount $BLOCKDEV
}

stop_tests () {
	cleanup_before_exit
	exit 1
}

check_and_mount_device () {

	# nothing to be done in FSMODE.
	[ -n "$FSMODE" ] && return

	cat /proc/mounts | grep "$MNTPOINT" > /dev/null
	if [ $? -eq 0 ] ; then
		echo "$MNTPOINT is in use. Will unmount and use."
		umount "$MNTPOINT"
		[ $? -ne 0 ] && echo "Unmount failed" && exit 1
	fi

	mkdir -p $MNTPOINT

	# See if device/partition to test is already mounted
	cat /proc/mounts | grep "$BLOCKDEV" > /dev/null
	if [ $? -eq 0 ] ; then
		echo "$BLOCKDEV is already mounted. Will remount"
		umount "$BLOCKDEV"
		[ $? -ne 0 ] && echo "Unmount failed" && exit 1
	fi

	mount $BLOCKDEV $MNTPOINT

	if [ $? -ne 0 ];then
		echo "Mount Failure"
		exit 1
	fi
}

# Pass a workload string and see if it is part of RUN_WORKLOADS or not and
# based on that decide whether to run or print stats of that workload.
# 0 --> matched, 1--> did not match
is_workload_in_run_workloads () {
	local workload=$1
	local item
	local IFS_OLD=$IFS
	IFS=" "

	for item in $RUN_WORKLOADS
	do
		if [ "$item" == "$workload" ];then
			IFS=$IFS_OLD
			return 0
		fi
	done

	# nothing matched.
	IFS=$IFS_OLD
	return 1
}

# 0 --> supported, 1--> unsupported
is_supported_workload () {
	local workload=$1
	local IFS_OLD=$IFS
	IFS=" "

	for index in ${!WORKLOAD_MAP_ARRAY[@]}
	do
		if [ "$index" == "$workload" ];then
			IFS=$IFS_OLD
			return 0
		else
			continue
		fi
	done

	# nothing matched.
	IFS=$IFS_OLD
	return 1
}

# Verify if all the workloads in the input string are valid or not
verify_workloads () {
	local workloads=$1

	for item in $workloads;do
		is_supported_workload "$item"
		if [ $? -eq 1 ];then
			# Some workload is not supported
			return 1
		fi
	done

	# Everything is fine. All the workloads in input string are valid
	return 0
}

# parameters
# workload--> whole custom workload string
# result --> sets it to only custom workload name and removes components
#
verify_prepare_custom_workload () {
	local workload=$1
	local __resultvar=$2
	local IFS_OLD=$IFS
	local custom_wl
	local component_wls
	local wl

	# If it is a custom workload, then atleast one : should be there
	echo $workload | grep -q ":"
	[ $? -ne 0 ] && return 1

	# Extract custom_wl string and verify component wl are valid
	custom_wl=`echo $workload | cut -d ":" -f1`
	# what is the positin of first : in string
	local first_colon_index=`expr index $workload $":"`
	# extract substring at the index
	component_wls=${workload:$first_colon_index}

	# There should be atleast one component wl
	[ "$component_wls" == "" ] && return 1

	# custom_wl can't have same name as one of inbuilt workloads
	is_supported_workload "$custom_wl"
	[ $? -eq 0 ] && return 1

	IFS=:
	for wl in $component_wls;do
		is_supported_workload "$wl"
		[ $? -ne 0 ] && IFS=$IFS_OLD && return 1
	done

	# all the component workloads of custom workload seem to be fine.
	# This workload needs to be added dynamically to the MAP array
	# Replace all : with space in component wls
	component_wls=`echo $component_wls | sed 's/:/ /g'`
	WORKLOAD_MAP_ARRAY[$custom_wl]="$component_wls"
	[ -n "$DEBUG" ] && echo "Added [$custom_wl] ["$component_wls"] to workload map array" >&2
	IFS=$IFS_OLD
	# return the result
	eval $__resultvar="'$custom_wl'"
	return 0
}


# Verify workloads. Also parse any custom workloads strings and modify
# RUN_WORKLOADS accordingly.
# parameters
# workloads --> full workload string
# results --> returns the workloads to run string after parsting it
verify_and_prepare_workload_string () {
	local workloads=$1
	local __resultvar=$2
	local IFS_OLD=$IFS
	local wl
	local custom_wl
	local run_workloads
	# Note, I am prefixing this variable with function name abbreviated.
	# the reason being that any caller will not get the right result
	# if caller happened to use the same variable name. Hence for any
	# result variables, prefix it with function name to avoid conflict
	local vapws_result
	IFS=,

	for wl in $workloads;do
		is_supported_workload "$wl"
		# If inbuilt workload, just append it to RUN_WORKLOADS. If it
		# is a custom one, we need more processing.
		if [ $? -eq 0 ];then
			if [ -n "$run_workloads" ];then
				run_workloads="$run_workloads $wl"
			else
				run_workloads="$wl"
			fi
			continue
		fi

		# See if this is custom workload
		verify_prepare_custom_workload $wl vapws_result
		if [ $? -eq 0 ];then
			if [ -n "$run_workloads" ];then
				run_workloads="$run_workloads $vapws_result"
			else
				run_workloads="$vapws_result"
			fi

			continue
		fi

		# Bad workload
		IFS=$IFS_OLD
		return 1
	done

	IFS=$IFS_OLD
	# Return the results.
	eval $__resultvar="'$run_workloads'"
}

map_workload_to_fiofile_outputfile () {
	local workload=$1
	local group

	# suffix "group" if group mode is enabled
	[ -n "$GROUPMODE" ] && group="-group"

	OUTPUTFILE="$workload-$IOSCHED$group.txt"
	FIOFILE="fio-$workload-$IOSCHED$group.job"
}

list_available_workloads () {
	# Note: ! helps with getting indexs and not value of index
	for index in ${!WORKLOAD_STRING_ARRAY[@]};do
		printf "%-12s%s\n" $index "${WORKLOAD_STRING_ARRAY[$index]}"
	done
}

generate_fio_global_section () {
	local fiofile=$1

	# Note: This overwrites the existing file
        echo "[global]" > $OUTPUTDIR/$fiofile
        echo "directory=$TESTDIR" >> $OUTPUTDIR/$fiofile
        echo "runtime=$RUNTIME" >> $OUTPUTDIR/$fiofile
#       echo "ioscheduler=$IOSCHED" >> $OUTPUTDIR/$fiofile
        echo "time_based=1" >> $OUTPUTDIR/$fiofile
        echo "size=$FILESIZE" >> $OUTPUTDIR/$fiofile
        echo "group_reporting=1" >> $OUTPUTDIR/$fiofile
        echo "bs=$BLOCKSIZE" >> $OUTPUTDIR/$fiofile
        echo "exec_prerun='echo 3 > /proc/sys/vm/drop_caches'" >> $OUTPUTDIR/$fiofile
	# If fio supports cgroup_nodelete option, then make sure cgroups are
	# not deleted after job completion so that we can capture blkio.time
	# data after job completion.
	local cgnodelete=`fio --cmdhelp | grep -w "cgroup_nodelete" | wc -l`
	if [ "$cgnodelete" != "0" ];then
        	echo "cgroup_nodelete=1" >> $OUTPUTDIR/$fiofile
	fi
        echo >> $OUTPUTDIR/$fiofile
}

__generate_fio_job_section () {
	local workload=$1
	local fiofile=$2
	local numjobs=$3
	local cgroup=$4
	local cgroup_weight=$5

	# In group mode, append name of cgroup to jobname so that output of
	# differnent cgroup jobs can be distinguished.

	# Use "iostest" as job name for all the jobs. This will force fio
	# to generate files like iostest.1.0 iostest.2.0 etc... This way
	# we can re-use same files across runs without bloating too much
	# Use $workload-$cgroup, like bsr-test1 in "description" string. This
	# description string will be printed in --minimal format after the
	# actual results. Use it to determine actual jobname and group.

	echo "[iostest]" >> $OUTPUTDIR/$fiofile

	if [ -n "$GROUPMODE" ];then
		echo "description=$workload-$cgroup" >>  $OUTPUTDIR/$fiofile
	else
		echo "description=$workload" >>  $OUTPUTDIR/$fiofile
	fi

	echo "numjobs=$numjobs" >>  $OUTPUTDIR/$fiofile

	# Launch every job in its own group. Helpful in mixed workloads
	echo "new_group=1" >>  $OUTPUTDIR/$fiofile

        if [ "$workload" == "bsr" ];then
                echo "rw=read" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "brr" ];then
                echo "rw=randread" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "brrmmap" ];then
                echo "rw=randread" >> $OUTPUTDIR/$fiofile
                echo "ioengine=mmap" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "bufw" ];then
                echo "rw=write" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "bufwfs" ];then
                echo "rw=write" >> $OUTPUTDIR/$fiofile
                echo "end_fsync=1" >> $OUTPUTDIR/$fiofile
	elif [ "$workload" == "bufwfs32" ];then
		echo "rw=write" >> $OUTPUTDIR/$fiofile
		echo "fsync=32" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "osyncw" ];then
                echo "rw=write" >> $OUTPUTDIR/$fiofile
                echo "sync=1" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "drr" ];then
                echo "rw=randread" >> $OUTPUTDIR/$fiofile
                echo "direct=1" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "drw" ];then
                echo "rw=randwrite" >> $OUTPUTDIR/$fiofile
                echo "direct=1" >> $OUTPUTDIR/$fiofile
                echo "overwrite=1" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "arr" ];then
		echo "ioengine=libaio" >> $OUTPUTDIR/$fiofile
		echo "iodepth=32" >> $OUTPUTDIR/$fiofile
                echo "rw=randread" >> $OUTPUTDIR/$fiofile
                echo "direct=1" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "arw" ];then
		echo "ioengine=libaio" >> $OUTPUTDIR/$fiofile
		echo "iodepth=32" >> $OUTPUTDIR/$fiofile
                echo "rw=randwrite" >> $OUTPUTDIR/$fiofile
                echo "direct=1" >> $OUTPUTDIR/$fiofile
                echo "overwrite=1" >> $OUTPUTDIR/$fiofile
        elif [ "$workload" == "drrmmap" ];then
                echo "rw=randread" >> $OUTPUTDIR/$fiofile
                echo "direct=1" >> $OUTPUTDIR/$fiofile
                echo "ioengine=mmap" >> $OUTPUTDIR/$fiofile
        fi

	#Output cgroup information. In ROOTGRP mode, we run in root group.
	if [ -n "$GROUPMODE" ] && [ "$ROOTGRP" == "" ];then
		echo "cgroup=$cgroup" >> $OUTPUTDIR/$fiofile
		echo "cgroup_weight=$cgroup_weight" >> $OUTPUTDIR/$fiofile
	fi

        echo >> $OUTPUTDIR/$fiofile
}

generate_fio_job_section_hetrogenous_groups() {
	local workload=$1
	local fiofile=$2
	local numjobs=$3
	local weight
	local job
	local j
	local group_data_entry
	local wl_full
	local grpname

	for grpname in ${!GROUP_DATA_ARRAY[*]}
	do
		group_data_entry=${GROUP_DATA_ARRAY[$grpname]}
		weight=`echo $group_data_entry | awk -F '#' '{print $1}'`
		wl=`echo $group_data_entry | awk -F '#' '{print $2}'`

		[ -n "$DEBUG" ] && echo "Generating fio job section for" \
			 " grp=$grpname weight=$weight wl=$wl numjobs=$numjobs"

		for job in `component_jobs_of_workload $wl`
		do
			__generate_fio_job_section "$job" "$fiofile" \
					"$numjobs" "$grpname" "$weight"
		done
	done
}

generate_fio_job_section_group () {
	local workload=$1
	local fiofile=$2
	local numjobs=$3
	local weight
	local job
	local j

	if [ -n "$HETROGENOUS_GROUPS" ];then
		generate_fio_job_section_hetrogenous_groups "$workload" \
				"$fiofile" "$numjobs"
		return
	fi

	# In group mode replicate the job section for each cgroup
	for((j=1;j<=$NRGRP;j++));do
		let weight=$CGBASEWEIGHT*$j
		for job in `component_jobs_of_workload $workload`
		do
			__generate_fio_job_section "$job" \
				"$fiofile" "$numjobs" "$CGBASESTRING$j"\
				 "$weight"
		done
	done
}

generate_fio_job_section_nogroup () {
	local workload=$1
	local fiofile=$2
	local numjobs=$3
	local job

	# non cgroup mode
	for job in `component_jobs_of_workload $workload`
	do
		[ -n "$DEBUG" ] && echo "Generating fio section for" \
					" job $job"
		__generate_fio_job_section "$job" "$fiofile" "$numjobs"
	done
}

generate_fio_job_section () {
	local workload=$1
	local fiofile=$2
	local numjobs=$3
	local j
	local weight


	if [ -n "$GROUPMODE" ];then
		generate_fio_job_section_group "$workload" "$fiofile" "$numjobs"
	else
		# non cgroup mode
		generate_fio_job_section_nogroup "$workload" "$fiofile" \
						"$numjobs"
	fi
}

do_generic_workload () {
	local workload=$1
	local numjobs=$2
	local outputfile=$3

	[ -n "$DEBUG" ] && echo "Generating fio file for workload $workload"
	generate_fio_global_section $FIOFILE
	generate_fio_job_section "$workload" "$FIOFILE" "$numjobs"
	fio --minimal $OUTPUTDIR/$FIOFILE >> $OUTPUTDIR/$OUTPUTFILE
}

# reset cgroup stats by changing ioscheduler
reset_cgroup_stats () {
	# Change elevator at the devices
	change_ioscheduler "noop"
	change_ioscheduler "$IOSCHED"
}

set_record_scheduler_parameters () {
	outputfile=$1

	# Also set group isolation mode
	if [ "$IOSCHED" == "cfq" ];then
		[ -n $GRPISOLATION ] && set_iosched_parameter "group_isolation" "$GRPISOLATION"
		echo "GRPISOLATION=$GRPISOLATION" >> $OUTPUTDIR/$outputfile

		[ -n "$SLICE_IDLE" ] && set_iosched_parameter "slice_idle" "$SLICE_IDLE"
		echo "SLICE_IDLE=$SLICE_IDLE" >> $OUTPUTDIR/$outputfile

		[ -n "$GROUP_IDLE" ] && set_iosched_parameter "group_idle" "$GROUP_IDLE"
		echo "GROUP_IDLE=$GROUP_IDLE" >> $OUTPUTDIR/$outputfile

		[ -n "$QUANTUM" ] && set_iosched_parameter "quantum" "$QUANTUM"
		[ -f "/sys/dev/block/$BLOCKDEVMAJOR:$BLOCKDEVMINOR/queue/iosched/quantum" ] && echo "QUANTUM=`cat /sys/dev/block/$BLOCKDEVMAJOR:$BLOCKDEVMINOR/queue/iosched/quantum`" >> $OUTPUTDIR/$outputfile
	fi
}

# After every test, record the blkio.time stats in outputfile.
record_cgtime_in_output_file () {
	local outputfile=$1
	local major=$2
	local minor=$3
	local j
	local cgtime

	# Do this only if group mode is enabled.
	[ -z "$GROUPMODE" ] && return

	# The format is cgtime;group1time;group2time;....
	echo -n "cgtime;" >> $OUTPUTDIR/$outputfile

	for((j=1;j<=NRGRP;j++))	;do
		cgtime=`cat $CGROUPMNTPOINT/$CGBASESTRING$j/blkio.time 2>/dev/null | grep "$major:$minor" | awk '{print $2}'`
		echo -n "$cgtime;" >> $OUTPUTDIR/$outputfile
	done
	echo >> $OUTPUTDIR/$outputfile
}

#Helper functions
do_test () {
	local testname=$1
	local nrset=$2
	local numjobs=$3
	local outputfile=$4

	echo "Starting test for [$testname] with set=$nrset numjobs=$numjobs filesz=$FILESIZE bs=$BLOCKSIZE runtime=$RUNTIME"

	reset_cgroup_stats
	set_record_scheduler_parameters "$outputfile"
	save_nrset_in_output_file "$outputfile" "$nrset"
	save_numjobs_in_output_file "$outputfile" "$numjobs"

	do_generic_workload "$workload" "$numjobs" "$outputfile"

	record_cgtime_in_output_file "$outputfile" "$BLOCKDEVMAJOR" "$BLOCKDEVMINOR"
}

setup_output_file () {
	outputfile=$1

	if [ -f "$OUTPUTDIR/$outputfile" ];then
		mv $OUTPUTDIR/$outputfile $OUTPUTDIR/$outputfile.bak
	fi

	touch $OUTPUTDIR/$outputfile
}

#cg_assign_group_weights () {
#	local j
#	local weight
#
       # By default assign increasing weights(100,200,300...)
#	for((j=1;j<=$NRGRP;j++));do
#		let weight=100*$j
#		echo $weight > $GROUPMNTPOINT/test$j/blkio.weight
#		if [ $? -ne 0 ];then
#			echo "Cgroup weight assignment failed"
#			stop_tests
#		fi
#	done
#}

mount_blkio () {
	local j
	local blkiomounted
	local cgmntpnt

	# Do this only if group mode is enabled.
	[ -z "$GROUPMODE" ] && return

	# mount blkio
	mkdir -p $CGROUPMNTPOINT

	# Check if blkio is already mounted. If yes, try to umount it.
	blkiomounted=`cat /proc/mounts | awk '{print $4}' | grep "blkio"`

	if [ "$blkiomounted" != "" ];then
		cgmntpnt=`cat /proc/mounts | grep "$blkiomounted" | awk '{print $2}'`
		echo "Blkio is already mounted at $cgmntpnt. Unmounting it"
		umount $cgmntpnt
		if [ $? -ne 0 ];then
			echo "Unmounting $cgmntpnt failed. Exiting"
			stop_tests
		fi
	fi

	mount -t cgroup -o blkio none $CGROUPMNTPOINT >> /dev/null
	if [ $? -ne 0 ];then
		echo "Mounting blkio cgroup Failed"
		stop_tests
	fi
}

misc_initializations () {
	mkdir -p $OUTPUTDIR
	[ $? -ne 0 ] && echo "Can't create dir $OUTPUTDIR" && exit 1

	# if CLEANUP specified, delete old fio and result files in OUTPUTDIR

	if [ "$CLEANUP" == "1" ]; then
		echo "Cleaning up $OUTPUTDIR"
		rm -f $OUTPUTDIR/*.job
		rm -f $OUTPUTDIR/*.txt
		rm -f $OUTPUTDIR/*.txt.bak
	fi
	check_and_mount_device
	mkdir -p $TESTDIR
	# install trap handler
	trap 'stop_tests' SIGINT

	# Mount blkio controller
	mount_blkio

	# Set IOscheduler on the devices
	change_ioscheduler "$IOSCHED"
}

# Given a workload string, construct a full wl string.
# example: desktop ---> desktop:bsr:brr:bufw
#          bsr--->bsr
#
get_full_wl_string () {
	local workload=$1
	local component_wls
	local colon_sep_component_wls

	component_wls=`component_jobs_of_workload "$workload"`
	if [ "$component_wls" == "$workload" ];then
		# This is regular inbuilt workload
		echo "$workload"
	else
		# This is composite workload
		colon_sep_component_wls=`echo $component_wls | sed 's/ /:/g'`
		echo "$workload:$colon_sep_component_wls"
	fi
}

save_workload_parameters_in_output_file () {
	local workload=$1
	local outputfile=$2
	local component_wls
	local colon_sep_component_wls
	local wl_full
	local weight
	local wl
	local group_data_entry

	# hostname
	echo "Host=`uname -n`" >> $OUTPUTDIR/$outputfile

	# kernel version
	echo "Kernel Version=`uname -r`" >> $OUTPUTDIR/$outputfile

	# fio version
	echo "FIOVERSION=`fio --version | awk '{print $2}'`" >> $OUTPUTDIR/$outputfile

	# Save Device and Dir we are testing on
	echo "DEVICE=$BLOCKDEV" >> $OUTPUTDIR/$outputfile
	echo "TESTDIR=$TESTDIR" >> $OUTPUTDIR/$outputfile

	# Group mode and nr groups
	if [ -n "$GROUPMODE" ];then
		echo "GROUPMODE=1" >> $OUTPUTDIR/$outputfile
		echo "NRGRP=$NRGRP" >> $OUTPUTDIR/$outputfile
		if [ -n "$HETROGENOUS_GROUPS" ];then
			echo "HETROGENOUS_GROUPS=1" >> $OUTPUTDIR/$outputfile
		fi
	fi

	# Save what workload we are running
	if [ -n "$HETROGENOUS_GROUPS" ];then
		# In hetrogenous mode, this is more of a jobname thing.
		echo "WORKLOAD=$workload" >> $OUTPUTDIR/$outputfile

		# Save each group's data. Format is.
		# GRP-<grpname>=<grpdata>
		# <grpdata> = weight#full_wl_string
		#
		for grpname in ${!GROUP_DATA_ARRAY[*]}
		do
			group_data_entry=${GROUP_DATA_ARRAY[$grpname]}
			weight=`echo $group_data_entry \
				| awk -F '#' '{print $1}'`
			wl=`echo $group_data_entry | awk -F '#' '{print $2}'`
			wl_full=`get_full_wl_string "$wl"`
			echo "GRP-$grpname=$weight#$wl_full" >> $OUTPUTDIR/$outputfile
		done
	else
		wl_full=`get_full_wl_string "$workload"`
		echo "WORKLOAD=$wl_full" >> $OUTPUTDIR/$outputfile
	fi

	# Log the fact that cgroup times have been recorded
	echo "CGTIME=1" >> $OUTPUTDIR/$outputfile

	# File Size
	echo "Filesz=$FILESIZE" >> $OUTPUTDIR/$outputfile
	echo "bs=$BLOCKSIZE" >> $OUTPUTDIR/$outputfile
}

save_numjobs_in_output_file () {
	local outputfile=$1
	local numjobs=$2

	echo "numjobs=$numjobs" >> $OUTPUTDIR/$outputfile
}

save_nrset_in_output_file () {
	local outputfile=$1
	local nrset=$2

	echo "Set=$nrset" >> $OUTPUTDIR/$outputfile
}

run_workload_set () {
	local workload=$1
	local nrset=$2
	local outputfile=$3
	local i=0

	if [ -n "$NRPROCS" ];then
		sync
		echo 3 > /proc/sys/vm/drop_caches
		do_test "$workload" "$nrset" "$NRPROCS" "$outputfile"
	else
		for((i=1;i<=$MAXTHREADS;i=i*2));do
			sync
			echo 3 > /proc/sys/vm/drop_caches
			do_test "$workload" "$nrset" "$i" "$outputfile"
		done
	fi
}

run_workload () {
	local workload=$1
	local i=0

	map_workload_to_fiofile_outputfile "$workload"
	setup_output_file $OUTPUTFILE
	save_workload_parameters_in_output_file "$workload" "$OUTPUTFILE"

	for ((i=1;i<=$NRSETS;i++));do
		run_workload_set "$workload" "$i" "$OUTPUTFILE"
	done

	echo "Finished test for workload [$workload]"
}

# Reporting logic follows
report_print_common_table_header () {
	local groupmode=$1
	local nrgrp=$2
	local hostname=$3
	local workload=$4
	local iosched=$5
	local kver=$6
	local filesize=$7
	local blocksize=$8
	local device=$9
	local testdir=${10}
	local group_isolation=${11}
	local slice_idle=${12}
	local group_idle=${13}
	local quantum=${14}
	local hetrogenous_groups=${15}

	printf "%-30s %-30s\n" "Host=$hostname" "Kernel=$kver"

	if [ -n "$groupmode" ];then
 		printf "%-20s %-20s" "GROUPMODE=1" "NRGRP=$nrgrp"
		if [ -n "$hetrogenous_groups" ];then
 			printf "%-30s" "HETROGENOUS_GROUPS=1"
		fi
		printf "\n"
	fi

	printf "%-30s %-30s\n" "DIR=$testdir" "DEV=$device"
	printf "%-18s%-16s%-12s%-8s\n" "Workload=$workload" "iosched=$iosched" "Filesz=$filesize" "bs=$blocksize"
	if [ "$iosched" == "cfq" ];then
		printf "%-18s%-16s%-16s%-12s \n" "group_isolation=$group_isolation" "slice_idle=$slice_idle" "group_idle=$group_idle" "quantum=$quantum"
	fi
	printf "=========================================================================\n"
}

print_process_nogroup_results () {
	local file=$1
	local workload=$2

	grep -v "^$" $file | awk -F\; -v workload=$workload -v fioparsemethod=$FIOPARSEMETHOD -v setdata=$SETDATA -v mergerw=$MERGERW -v noheader=$NOHEADER '

	function awk_max (val1, val2) {
		if (val1 >= val2)
			return val1
		else
			return val2
	}

	# lformat is local variable. Rest are parameters
	function print_nogrp_table_header(lformat) {
		if (noheader)
			return

		if (mergerw) {
			lformat="%-10s%-4s%-4s%-15s%-15s\n"
			printf lformat, "job", "Set", "NR", "BW(KB/s)", "MaxClat(us)"
	  		printf lformat, "---", "---", "--","------------", "-----------"
		} else {
	  		lformat="%-10s%-4s%-4s%-15s%-15s%-15s%-15s\n"
			printf lformat, "job", "Set", "NR", "ReadBW(KB/s)", "MaxClat(us)", "WriteBW(KB/s)", "MaxClat(us)"
	  		printf lformat, "---", "---", "--","------------", "-----------", "-------------", "-----------"
		}
	}

	function print_nogrp_table_value(jobname, nrset, numjobs, readbw, readmaxlat, writebw, writemaxlat,   lformat) {
		if (mergerw) {
			lformat="%-10s%-4s%-4s%-15s%-15s\n"
	  		printf lformat, jobname,nrset,numjobs,readbw+writebw,awk_max(readmaxlat,writemaxlat)
		} else {
	  		lformat="%-10s%-4s%-4s%-15s%-15s%-15s%-15s\n"
	  		printf lformat, jobname,nrset,numjobs,readbw,readmaxlat,writebw,writemaxlat;
		}
	}

	BEGIN {
	  if (setdata)
	  	print_nogrp_table_header()
	}
	{
	  # If a line starts with ";", it is description line. This will
	  # the actual job name and group name. Ex. bsr-test1. Because
	  # we have used -F; for this line NF=2 with $1="" $2=bsr-test1

	  if (NF==2 && $1=="") {
		# This is description line starting with ";". Extract jobname.
		jobname=$2

		# We got jobname. Data related to jobname we must have read
		# in previous line. Process the data now.
		if (setdata)
	  		print_nogrp_table_value(jobname,nrset,numjobs,readbw,readmaxlat,writebw,writemaxlat)

		# Store some data in array. Prefix it with some increasing
		# number so that we can sort indices later while traversing
		# the array
		readbwarray[jobcounter,jobname,numjobs]+=readbw
		readlatarray[jobcounter,jobname,numjobs]+=readmaxlat
		writebwarray[jobcounter,jobname,numjobs]+=writebw
		writelatarray[jobcounter,jobname,numjobs]+=writemaxlat
	  } else if (NF < 10) {
		# Traverse through sets
		if (match($1,"Set=")) {
			oldnrset=nrset
			nrset=substr($1,RSTART+RLENGTH);
			if (!match(nrset, oldnrset)) {
				# Print a new line after set completion
				if (setdata)
					printf "\n"
				# Reset the job counter
				jobcounter=0
			}
		}

		if (match($1,"numjobs=")) {
			numjobs=substr($1,RSTART+RLENGTH);

			# This counter goes up by one as soon as all the sub
			# jobs in a workload are over. For example, when bsr
			# brr and bufw for workload "desktop" are over for
			# a particular number of jobs, this counter goes up.
			jobcounter++
		}

	  } else {
		#Assign values to readable variables. Just store the data
		# values in variables. These will be printed when we read
		# jobname from next line which is a description line.
		if (fioparsemethod == 1) {
			readbw=$5
			readmaxlat=$12
			writebw=$21
			writemaxlat=$28
		} else if (fioparsemethod == 2) {
			# Total lat fields have been added
			readbw=$5
			readmaxlat=$12
			writebw=$25
			writemaxlat=$32
		} else if (fioparsemethod == 3) {
			# Total lat fields have been added and also a
			# versioning field has been added
			readbw=$6
			readmaxlat=$13
			writebw=$26
			writemaxlat=$33
		}
	  }
	}
	END {
		if (!noheader) {
			printf "\n%-16s\n", "AVERAGE[" workload "]"
			printf "%-8s\n", "-------"
		} else if (setdata) {
			# If setdata was printed, introduce a new line to
			# be able to separate out average.
			printf "\n"
		}

		print_nogrp_table_header()

		# sort the source array using index. sorted array will
		# be stored in dest. dest will now contain source
		# indeces sorted.
		n = asorti(readbwarray, dest)

		# Walk through all elements of array and print
		for (i = 1; i <= n; i++) {
			split(dest[i], separate, SUBSEP)
			print_nogrp_table_value(separate[2], nrset, separate[3], readbwarray[dest[i]]/nrset, readlatarray[dest[i]]/nrset, writebwarray[dest[i]]/nrset, writelatarray[dest[i]]/nrset)
		}
	}'
}

__print_process_group_results () {
	local file=$1
	local wl_full=$2
	local nrsets=$3
	local nrgrp=$4

	# THE BIG awk parsing script
	grep -v "^$" $file | awk -F\; -v workloadfull="$wl_full" -v nrsets=$nrsets -v nrgrp=$nrgrp -v cgbasestring=$CGBASESTRING -v cgtime=$CGTIME -v fioparsemethod=$FIOPARSEMETHOD -v setdata=$SETDATA -v total=$TOTAL -v debug=$DEBUG '
	  # Note: grpstring and i are local variables and following function
	  # does not take any arguments.

	  function awk_print_column_headers (grpstring,i)  {
		printf format, "job", "Set", "NR"

		# Print columns for groups
		for (i = 1; i <= nrgrp; i++) {
		      grpstring=sprintf("%s%d", cgbasestring,i)
		      printf grpformat, grpstring
		}

		# if "total" print column for total
		if (total)
			printf grpformat, "total"

		printf "\n"

		printf format, "---", "---", "--"

		# Print columns for groups
		for (i = 1; i <= nrgrp; i++) {
		      printf grpformat, "-------"
		}

		if (total)
		      printf grpformat, "-------"
			
		printf "\n"
	}

	# Print total data of a test
	function print_total_data (totaldata) {
		if (!total)
			return

		printf grpformatint, totaldata
	}

	# Adds a jobname to jobnamearray. This job array is used to keep track
	# of how many jobs have been run and loop through these to print
	# report. (bsr, brr, drr...etc).
	# global-->jobnamearray
	# parameters
	# jobname --> jobname to add to array
	function add_entry_to_array (arrayname, data,   llen) {
		llen=length(arrayname)
		arrayname[llen]=data
	}

	# Adds an entry to job array if it does not already exist
	function search_add_entry_to_array (arrayname, data,   llen,li) {
		llen=length(arrayname)
		for(li = 0; li < llen; li++) {
			if(arrayname[li] == data)
				return
		}

		# Entry not found in array. add it.
		add_entry_to_array(arrayname, data)
	}

	# debug print of array
	#
	function debug_print_array_contents (arrayname, li) {

		printf "Contents of array %s\n", arrayname[0]
		for(li in arrayname)
			printf "%s[%s]=%s\n", arrayname[0], li, arrayname[li]
		printf "\n"
	}

	# Prints the data of a group.
	# globals (globalbwarray, grpformatint)
	#
	function print_group_data (stattype, setnr, jobname, numjobs, grpname, lgrpdata, li) {
		if (setnr == 0) {
			# We need to print average data of all sets
			for(li = 1; li <= nrsets; li++) {
				if (stattype == "cgtime")
					lgrpdata += global_cgtime_array[li,numjobs,grpname]
				else
					lgrpdata += globalbwarray[li,grpname,jobname,numjobs]
			}
			lgrpdata = lgrpdata/nrsets
		} else {
			if (stattype == "cgtime")
				lgrpdata=global_cgtime_array[setnr,numjobs,grpname]
			else
				lgrpdata=globalbwarray[setnr,grpname,jobname,numjobs]
		}

		printf grpformatint, lgrpdata
		return lgrpdata
	}

	# global (format, nrgrp)
	function print_job_data (stattype, setnr, numjobs, jobname,  li,ltotaldata) {
		# Print jobname, nrset, numjobs fields
		if (setnr == 0)
			# This is average data stats
			printf format, jobname, nrsets, numjobs
		else
			printf format, jobname, setnr, numjobs

		# Print data of each group now
		for(li = 1; li <= nrgrp; li++)
			ltotaldata+=print_group_data(stattype, setnr, jobname, numjobs, grpnamearray[li])
		# print total
		print_total_data(ltotaldata)
		# Finish the job line
		printf "\n"
	}

	# For a particular nrthread number, like 1, 2 4, ..., go through all
	# the component jobs for that thread number and print stats. This
	# makes sure that all the component jobs of a workload are together
	# for a particular number of threads.
	#
	function print_jobnum_data (stattype, setnr, jobnum,	llen,li) {
		llen=length(workloadarray)

		# ctime stats are per high level wl and not per component
		# job.
		if (stattype == "cgtime") {
			print_job_data(stattype, setnr, jobnum, workloadarray[1])
			return
		}

		# Process bw stats
		# index 0 contains name of array
		if(llen > 2) {
			for(li = 2; li < llen; li++)
				print_job_data(stattype, setnr, jobnum, workloadarray[li])
		} else
			print_job_data(stattype, setnr, jobnum, workloadarray[1])
	}

	# global (jobnumarray)
	#
	function print_set_data (stattype, setnr,  llen,li) {
		llen=length(jobnumarray)

		for(li = 1; li < llen; li++)
			print_jobnum_data(stattype, setnr, jobnumarray[li])
	}

	# Goes through globalbwarray and prints all set data for all groups.
	# globals (nrsets)
	#
	function print_all_set_data (stattype, li) {
		awk_print_column_headers()
		for(li = 1; li <= nrsets; li++) {
			print_set_data(stattype, li)
			printf "\n"
		}
	}

	# print avg data of sets
	#
	function print_avg_all_set_data (stattype) {
		if (stattype == "cgtime")
			printf "%-16s%-16s\n", "AVERAGE[" workload "]","[cgroup time in ms]"
		else
			printf "%-16s%-16s\n", "AVERAGE[" workload "]","[bw in KB/s]"

		printf "%-8s\n", "-------"
		awk_print_column_headers()
		# Note. This is a hack. Pass setnr=0 to signal that we want
		# average of three sets.
		print_set_data(stattype, 0)
		printf "\n"
	}

	# print cgtime stats
	# We can not  use globalbwarray functions to print these as these stats
	# are per high level workload and not per individual jobs inside
	# workload. This set of functions should be useful to print other
	# cgroup stats like nr requests, transferred bytes etc.
	#

	function print_cgtime_stats (li) {
		printf "cgroup time stats (ms)\n"
		printf "----------------------\n"
		print_all_set_data("cgtime")
	}

	#
	# process_cgtime_input_line
	# global (nrgrp, grpnamearray)
	#
	# This function assumes that grpnamearray has been populated before
	# even a single cgtime line appears.
	#
	function process_cgtime_input_line (lgrpname,li) {
		# If $1=cgtime, it is cgroup time information. Note, this
		# this info is per workload and not per job. (for ex, if
		# workload is mixed, then it is not for bsr, brr, etc..).
		for (li = 1; li <= nrgrp; li++) {
			lgrpname=grpnamearray[li]
			global_cgtime_array[cursetnr,numjobs,lgrpname]=$(li+1)

			# Also save total time across sets to print avg
			# stats.
			global_cgtotaltime_array[numjobs,lgrpname]+=$(li+1)
		}
	}
	  BEGIN {
		if (debug) {
			printf "Start of BEGIN\n"
			printf "workloadfull=%s\n", workloadfull
		}
		# format for printing jobname, nrset, jobnum.
		format="%-8s%-4s%-4s"
		grpformat="%-7s"

		# for printing bw stats of group.
		grpformatint="%-7d"

		# Parse workloadfull and set workload. Note: workload is
		# high level workload string (in case of composite workload)
		# workloadfull=desktop:bsr:brr:bufw   workload=desktop
		#
		split(workloadfull,workloadarray,":")

		# Store name of array at 0 index
		workloadarray[0]="workloadarray"
		workload=workloadarray[1]

		if (debug)
			debug_print_array_contents(workloadarray)

		# Create empty arrays, otherwise length() function complains.
		# Also it is useful to store array name in index 0.
		grpnamearray[0]="grpnamearray"
		jobnumarray[0]="jobnumarray"
		global_cgtime_array[0]="global_cgtime_array"
		global_cgtotaltime_array[0]="global_cgtotaltime_array"
		globalbwarray[0]="globalbwarray"
	}

	# Main body of awk
	#
	{
	  # If a line starts with ";", it is description line. This will
	  # the actual job name and group name. Ex. bsr-test1. Because
	  # we have used -F; for this line NF=2 with $1="" $2=bsr-test1

	  if (NF==2 && $1=="") {
		# This is description line starting with ";". Extract jobname
		# and group name
		# split job and group name
		split($2, jobgroupname, "-")

		if (debug)
			printf "Adding jobname=%s grpname=%s numjobs=%d to arrays\n", jobgroupname[1], jobgroupname[2], numjobs

		# Add jobname to job array
		search_add_entry_to_array(grpnamearray,jobgroupname[2])
		search_add_entry_to_array(jobnumarray,numjobs)

		# Save the group data in the array.
		# Array index fields are
		# nrset, groupname, jobname, numjobs
		globalbwarray[cursetnr,jobgroupname[2],jobgroupname[1],numjobs]=totalbw
	  } else if ($1 == "cgtime") {
		process_cgtime_input_line()
	  } else if (NF < 10) {
		# Traverse through sets and numjobs lines
		if (match($1,"Set="))
			cursetnr=substr($1,RSTART+RLENGTH);

		if (match($1,"numjobs="))
			numjobs=substr($1,RSTART+RLENGTH);
	  } else {
		# This is the actual data related to the job like "iostest".
		# read it and store in variables. Printing of this data will
		# happen once the next line has been parsed which contains
		# the actual jobname and group name bsr-test1.
		if (fioparsemethod == 1) {
			readbw=$5
			writebw=$21
			totalbw=readbw+writebw
		} else if (fioparsemethod == 2) {
			# Total lat fields have been added
			readbw=$5
			writebw=$25
			totalbw=readbw+writebw
		} else if (fioparsemethod == 3) {
			# Total lat fields have been added and also a
			# versioning field has been added
			readbw=$6
			writebw=$26
			totalbw=readbw+writebw
		}
	  }
	}
	END {
		if (debug) {
			printf "Start of END\n"
			debug_print_array_contents(grpnamearray)
			debug_print_array_contents(jobnumarray)
			debug_print_array_contents(globalbwarray)
			debug_print_array_contents(global_cgtime_array)
			debug_print_array_contents(global_cgtotaltime_array)
		}

		if (setdata)
			print_all_set_data("bw")

		# Print avg bw stats
		print_avg_all_set_data("bw")

		# Also print cgroup time stats
		if (cgtime) {
			if (setdata)
				print_cgtime_stats()
			print_avg_all_set_data("cgtime")
		}
	}'
}

print_process_group_results () {
	local file=$1
	local workload=$2
	local nrsets=$3
	local nrgrp=`grep "NRGRP=" $file | cut -d "=" -f2`
	local wl_full
	local wl
	local temp

	# Parse what workload was run from the raw data file
	wl_full=`grep "^WORKLOAD=" $file | cut -d "=" -f2`
	wl=`echo $wl_full | awk -F : '{print $1}'`

	# Now call awk script to parse and print results.
	__print_process_group_results "$file" "$wl_full" "$nrsets" "$nrgrp"
}

__print_process_hetrogenous_group_results () {
	local file=$1
	local wl_full=$2
	local nrsets=$3
	local nrgrp=$4

	grep -v "^$" $file | awk -F \; -v workload="$wl_full" -v nrsets=$nrsets -v nrgrp=$nrgrp -v cgtime=$CGTIME -v fioparsemethod=$FIOPARSEMETHOD -v setdata=$SETDATA -v total=$TOTAL -v debug=$DEBUG '

 # Helper functions
 function hetro_print_column_headers (stattype,  lgrpname,lgrp_data_entry,
				lwl_full,lwl)  {

	# Printing first line of table
	# Print spaces above set information
	printf setformat, " "

	# Print group data related to wl and group name
	for (lgrpname in group_data_array) {
		if (lgrpname == 0)
			continue
		split(group_data_array[lgrpname],lgrp_data_entry,"#")
		# Now lgrp_data_entry[1]=weight lgrp_data_entryp[2]=full_wl
		#
		split(lgrp_data_entry[2], lwl_full, ":")
		lwl=lwl_full[1]

		printf grpname_wl_format, lgrpname, lgrp_data_entry[1],lwl
	}

	printf "\n"

	# Now print second line of table
	printf setformat, "Set"

	# Print columns for groups
	for (i = 1; i <= nrgrp; i++) {
		if (stattype == "bw")
			printf grp_job_NR_data_format, "job", "NR", "bw,KB/s"
	}
	printf "\n"

	# Now print --- under all the columns
	printf setformat, "---"
	# Print columns for groups (---)
	for (i = 1; i <= nrgrp; i++) {
		if (stattype == "bw")
			printf grp_job_NR_data_format, "---", "--", "-------"
	}
	printf "\n"
 }

 # Adds a jobname to jobnamearray. This job array is used to keep track
 # of how many jobs have been run and loop through these to print
 # report. (bsr, brr, drr...etc).
 # global-->jobnamearray
 # parameters
 # jobname --> jobname to add to array
 function add_entry_to_array (arrayname, data,   llen) {
	llen=length(arrayname)
	arrayname[llen]=data
 }

 # Adds an entry to job array if it does not already exist
 function search_add_entry_to_array (arrayname, data,   llen,li) {
	llen=length(arrayname)
	for(li = 0; li < llen; li++) {
		if(arrayname[li] == data)
			return
	}

	# Entry not found in array. add it.
	add_entry_to_array(arrayname, data)
 }

 # index array "arrayname" with "grpname" and append "data" to the existing
 # data of the group. Appending should be space separated. If no entry
 # exists for group, create new. If data is already part of the string,
 # then do not  add it again. For example, if we already recoded nrjob=1
 # for group test1, do not do it again. This can happen if with nrjob=1
 # we ran two component jobs, bsr and brr.
 #
 function search_append_entry_to_array (arrayname, grpname, data,   lentry,
				lentry_split, __lentry) {

	lentry=arrayname[grpname]

	if (lentry == "")
		arrayname[grpname]=data
	else {
		split(lentry,lentry_split," ")
		for (__lentry in lentry_split) {
			if (lentry_split[__lentry] == data)
				# data is already part of entry
				return
		}
		# data is not part of entry. Add it.
		arrayname[grpname]=sprintf("%s %s", lentry, data)
	}
 }

 # Go through each entry of an array. Each index data is a space separate
 # string. Count number of elements in each string and return the max number
 # of elements any index has got.
 # [test1] = "bsr brr"
 # [test2] = "bsr brr bufw"
 # here max entries are 3
 #
 function find_max_entries_of_any_array_index(arrayname, lret,li,llen,
				lentry_split) {
	for (li in arrayname) {
		if (li == 0)
			continue
		split(arrayname[li],lentry_split," ")
		if (length(lentry_split) > lret)
			lret=length(lentry_split)
	}
	return lret
 }

 # Takes an array, and index name. Retrieves the string and returns the
 # indexed component in string.
 # ex. array[test1]="bsr brr"
 #     array[test2]="bsr brr bufw"
 #  for index=test2, subindex=3, we will return bufw
 function get_array_index_element_subindex(arrayname, aindex, subindex,
			lentry_split) {
	split(arrayname[aindex],lentry_split, " ")
	if (subindex > length(lentry_split))
		return 0
	else
		return lentry_split[subindex]
 }

 # debug print of array
 #
 function debug_print_array_contents (arrayname, li) {
	printf "Contents of array %s\n", arrayname[0]
	for(li in arrayname)
		printf "%s[%s]=%s\n", arrayname[0], li, arrayname[li]
	printf "\n"
 }

 # Parse the GRP-<grpname> lines and store parsed data in an array.
 #
 function parse_raw_grpdata_lines (lgrpname,lgrpdata,ltemp) {
	if (match($1,"GRP-.*=")) {
		lgrpdata=substr($1,RSTART+RLENGTH);
		ltemp=substr($1,RSTART,RLENGTH);

		#ltemp should not contain "GRP-<grpname>=". Get rid of "GRP-"
		# and "=" to get to grpname
		sub("GRP-", "", ltemp)
		sub("=", "", ltemp)
		lgrpname=ltemp
	}

	group_data_array[lgrpname]=lgrpdata
 }

 # Prints the data of a group.
 #
 function print_group_data (stattype, setnr, jobnumsetidx, jobnamesetidx,
				grpname, lgrpdata,ljobnum,ljobname,li,ltemp) {
	ljobnum=get_array_index_element_subindex(group_jobnum_array,grpname,
				jobnumsetidx)
	ljobname=get_array_index_element_subindex(group_jobname_array,grpname,
				jobnamesetidx)
	if (ljobname == 0) {
		# This group does not have associated data. Nothing to do.
		# just print spaces.
		printf grp_job_NR_data_format, " ", " ", " "
		return
	}

	if (setnr == 0) {
		# Print average data
		for (li = 1; li <=nrsets; li++)
			ltemp+=group_bw_array[li,grpname,ljobname,ljobnum]
		lgrpdata = ltemp/nrsets
		# convert to integer and get rid of decimal part
		lgrpdata=sprintf("%d",lgrpdata)
	} else {
		lgrpdata=group_bw_array[setnr,grpname,ljobname,ljobnum]
	}

	printf grp_job_NR_data_format, ljobname, ljobnum, lgrpdata

	if (debug) {
		printf "print_group_data, setnr=%d grpname=%s ljobname=%s" \
			" ljobnum=%d lgrpdata=%d\n", setnr, grpname, ljobname,
			ljobnum, lgrpdata
	}
 }

 # Given a set, jobnumidx, jobnameidx, go thorough all the groups and print
 # particular stat
 #
 function print_componentidx_data (stattype, setnr, jobnumsetidx,
				jobnamesetidx, lgrpname) {
	if (debug) {
		printf "print_componentidx_data: setnr=%d jobnumsetidx=%d" \
			" jobnamesetidx=%d\n", setnr, jobnumsetidx,
			jobnamesetidx
	}

	if (setnr == 0)
		# This is average data print
		printf setformat, nrsets
	else
		printf setformat, setnr

	for (lgrpname in group_data_array) {
		if (lgrpname == 0)
			continue
		print_group_data(stattype, setnr, jobnumsetidx, jobnamesetidx,
			lgrpname)
	}

	# Finished printing single line of data
	printf "\n"
 }

 # For a particular jobnumset idx like first, second, third,, go through all
 # the groups and print their components.
 # First we determine the max number of components of a workload a group
 # might have run. Then we call that group that many times and group
 # prints one component at a time. these componets are indexed by component_idx.
 #
 function print_jobnumidx_data (stattype, setnr, jobnumsetidx,	llen,li,
				lmax_components) {

	if (debug) {
		printf "print_jobnumidx_data: setnr=%d jobnumsetidx=%d\n",
			setnr, jobnumsetidx
	}
	lmax_components=find_max_entries_of_any_array_index(group_jobname_array)

	for (li = 1; li <= lmax_components; li++) {
		print_componentidx_data(stattype, setnr, jobnumsetidx, li)
	}

	# Finshed printing data for one set of job numbers. Print a new line
	# This looks much nicer workload was composite.
	printf "\n"
 }

 # print_set_data
 # We try to keep track of how many instances of different nrjobs are there.
 # IOW, if a group has run workload bsr  for nrprocesses 1, 2, 4 and 8, then
 # there are 4 numjobs set samples. Number of samples for numjobs set should
 # be same for each group, that is a different thing that one group might
 # have run for 1, 2, 4 threads and other group might have run for 4, 8, 12
 # threads etc.
 #
 function print_set_data (stattype, setnr,  li,lnr_numjobs,lentry_split) {
	if (debug)
		printf "print_set_data: stattype=%s setnr=%d\n", stattype, setnr
	# get first entry from group_jobnum_array and count the numjobs
	# set samples.
	for (li in group_jobnum_array) {
		if (li == 0)
			continue
		split(group_jobnum_array[li],lentry_split," ")
		lnr_numjobs=length(lentry_split)
		break
	}

	# Print data for all numjobs set elements.
	for(li = 1; li <= lnr_numjobs; li++)
		print_jobnumidx_data(stattype, setnr, li)
 }

 # Prints all set data for all groups.
 # globals (nrsets)
 #
 function print_all_set_data (stattype, li) {
	hetro_print_column_headers(stattype)
	for(li = 1; li <= nrsets; li++) {
		print_set_data(stattype, li)
		# One set of data finished.
		printf "\n"
	}
 }

 # Prints all set data for all groups.
 # globals (nrsets)
 #
 function print_avg_set_data (stattype, li) {
	printf "%-16s%-16s\n", "AVERAGE[" workload "]","[bw in KB/s]"
	printf "%-8s\n", "-------"
	hetro_print_column_headers(stattype)
	# For average data, pass setnr=0
		print_set_data(stattype, 0)
		printf "\n"
 }

 BEGIN {
	if (debug)
		printf "BEGIN\n"

	# out of 80 columns, 4 chars for set dat and rest 76 chars are divided
	# equally among 4 groups (19 chars each).
	# out of 19, every group prints 3 columns. job(8), NR(3), data(8)
	setformat="%-4s"
	# grpname (6), weight(3), workload(8)
	grpname_wl_format="[%-5s,%-3s,%-7s]"
	grp_job_NR_data_format="%-8s%-3s%-8s"

	# contains user passed information about group like group weight,
	# workload to run etc
	group_data_array[0]="group_data_array"

	# One entry for each group. indexed by group name. data of each
	# group contains space separated number of jobs which have been
	# run for that group.
	# ex. [test1]="1 2 4 8"
	# test1 has run a workload with nr processes 1, 2, 4 8 respectively.
	#
	group_jobnum_array[0]="group_jobnum_array"

	# One entry for each group. indexed by group name. data of each
	# group contains space separated name jobs which have been
	# run for that group.
	# ex. [test1]="bsr brr bufw"
	# test1 has run a workload with components "bsr, brr and bufw".
	#
	group_jobname_array[0]="group_jobname_array"

	# contains per group total bw
	group_bw_array[0]="group_bw_array"
 }

# Main body of awk
 {
 # If a line starts with ";", it is description line. This will
 # the actual job name and group name. Ex. bsr-test1. Because
 # we have used -F; for this line NF=2 with $1="" $2=bsr-test1

 if (NF==2 && $1=="") {
 	# This is description line starting with ";". Extract jobname
	# and group name
	# split job and group name
	split($2, jobgroupname, "-")

	if (debug) {
		printf "Adding jobname=%s grpname=%s numjobs=%d to arrays\n",
			jobgroupname[1], jobgroupname[2], numjobs
	}
	# Add jobnum for a group to the array
	search_append_entry_to_array(group_jobnum_array, jobgroupname[2],
					numjobs)

	# Add jobname for a group to the array
	search_append_entry_to_array(group_jobname_array, jobgroupname[2],
					jobgroupname[1])

	# Save the group data in the array.
	# Array index fields are
	# nrset, groupname, jobname, numjobs
	group_bw_array[cursetnr,jobgroupname[2],jobgroupname[1],numjobs]=totalbw
 } else if (NF < 10) {
	# Traverse through sets and numjobs lines
	if (match($1,"Set="))
		cursetnr=substr($1,RSTART+RLENGTH);

	if (match($1,"numjobs="))
		numjobs=substr($1,RSTART+RLENGTH);

	# Traverse through GRP-<grpname> lines and build the array for
	# group names and their weights and workloads.
	# note, match() takes the regular expressions.
	#
	if (match($1,"GRP-.*=")) {
		parse_raw_grpdata_lines()
	}

 } else {
	# This is the actual data related to the job like "iostest".
	# read it and store in variables. Printing of this data will
	# happen once the next line has been parsed which contains
	# the actual jobname and group name bsr-test1.
	if (fioparsemethod == 1) {
		readbw=$5
		writebw=$21
		totalbw=readbw+writebw
	} else if (fioparsemethod == 2) {
		# Total lat fields have been added
		readbw=$5
		writebw=$25
		totalbw=readbw+writebw
	} else if (fioparsemethod == 3) {
		# Total lat fields have been added and also a
		# versioning field has been added
		readbw=$6
		writebw=$26
		totalbw=readbw+writebw
	}
 }
}
# End of main body of awk
 END {
	if (debug) {
		printf "END\n"
		debug_print_array_contents(group_data_array)
		debug_print_array_contents(group_jobnum_array)
		debug_print_array_contents(group_jobname_array)
		debug_print_array_contents(group_bw_array)
	}

	if (setdata)
		print_all_set_data("bw")

	print_avg_set_data("bw")

 }'
}

print_process_hetrogenous_group_results () {
	local file=$1
	local workload=$2
	local nrsets=$3
	local nrgrp=`grep "NRGRP=" $file | cut -d "=" -f2`
	local wl_full
	local wl
	local temp

	# Parse what workload was run from the raw data file
	wl_full=`grep "^WORKLOAD=" $file | cut -d "=" -f2`
	wl=`echo $wl_full | awk -F : '{print $1}'`

	# Now call awk script to parse and print results.
	__print_process_hetrogenous_group_results "$file" "$wl_full" "$nrsets" \
						 "$nrgrp"
}

parse_results () {
	local datadir=$1
	local groupmode
	local nrgrp
	local temp
	local workloadfull
	local workload
	local hetrogenous_groups

	for file in $datadir/*.txt*
	do
		# Ignore files of zero size
		[ ! -s "$file" ] && continue

		# If user specified a WL, print results for that workload only
		if [ -n "$WORKLOADS" ];then
			workload=`basename $file | cut -d "-" -f1`
			is_workload_in_run_workloads "$workload"
			if [ $? -eq 1 ];then
				continue;
			fi
		fi

		# Determine host/node name
		local hostname=`grep "^Host=" $file | cut -d "=" -f2`

		# Determine fio version used and set parse method
		local fioversion=`grep "^FIOVERSION=" $file | cut -d "=" -f2`
		set_fio_result_parse_method $fioversion

		# Determine workload name. Use WORKLOAD keyword in report file
		# workloadfull is workload name in full form. Including
		# component names, if any
		#
		workloadfull=`grep "^WORKLOAD=" $file | cut -d "=" -f2`
		workload=`echo $workloadfull | awk -F : '{print $1}'`

		# Determine iosched
		local iosched=`basename $file | cut -d "-" -f2 | cut -d "." -f1`

		# Determine kernel version
		local kver=`grep "Kernel Version" $file | cut -d "=" -f2`

		# Determine file size and block size
		local filesize=`grep "Filesz=" $file | cut -d "=" -f2`
		local blocksize=`grep "^bs=" $file | cut -d "=" -f2`

		# Determine device and testdir
		local device=`grep "^DEVICE=" $file | cut -d "=" -f2`
		local testdir=`grep "^TESTDIR=" $file | cut -d "=" -f2`

		# Determine if result file was generated in group mode
		groupmode=`grep "GROUPMODE=1" $file`
		hetrogenous_groups=`grep "HETROGENOUS_GROUPS=1" $file`
		nrgrp=`grep "NRGRP=" $file | cut -d "=" -f2`

		# Print iosched parameters
		local group_isolation=`grep "GRPISOLATION=" $file | head -1 | cut -d "=" -f2`
		local slice_idle=`grep "SLICE_IDLE=" $file | head -1 | cut -d "=" -f2`
		local group_idle=`grep "GROUP_IDLE=" $file | head -1 | cut -d "=" -f2`
		local quantum=`grep "QUANTUM=" $file | head -1 | cut -d "=" -f2`

		# Determine how many sets of data is there
		nrsets=`grep "Set=" $file | tail -1 | cut -d "=" -f2`

		[ "$NOHEADER" == "" ] && report_print_common_table_header "$groupmode" "$nrgrp" "$hostname" "$workload" "$iosched" "$kver" "$filesize" "$blocksize" "$device" "$testdir" "$group_isolation" "$slice_idle" "$group_idle" "$quantum" "$hetrogenous_groups"

		if [ "$groupmode" != "" ];then
			if [ "$hetrogenous_groups" == "" ];then
				print_process_group_results "$file" \
						"$workload" "$nrsets"
			else
				print_process_hetrogenous_group_results "$file"\
						"$workload" "$nrsets"
			fi
		else
			print_process_nogroup_results "$file" "$workload"
		fi

		echo
	done
}

generate_report () {
	local datadir=$1
	parse_results $datadir
}

diffreport_prepare_list_of_workloads () {
	local rawfile1=$1
	local rawfile2=$2
	local workloadsfile=$3
	local tempfile=`mktemp /tmp/iostest.tmp.XXXXX`

	# Determine what workloads we are going to generate reports for
	cat $rawfile1 | grep AVERAGE > $workloadsfile
	sed -i 's/AVERAGE\[//g' $workloadsfile
	sed -i 's/]//g' $workloadsfile

	sort $workloadsfile > $tempfile
	cp $tempfile $workloadsfile
	rm $tempfile
}

# Extract the data of a particular workload from a source file to a
# destination file.
diffreport_extract_workload_raw_data () {
	local srcfile=$1
	local dstfile=$2
	local workload=$3

	cat $srcfile | awk -v workload=$workload -v dstfile=$dstfile '{
		if (start_found) {
			# Start has been found. Keep on printing line till end
			# is encountered.
			# Look for end marker or EOF
			if (match($1, "Host=")) {
				start_found=0
			} else
				print $0 >> dstfile
		} else {
			# Note: Trying to match "[" is having issues with
			# match().
			matchstring=workload"]"
			if (match($1, "AVERAGE") && match($1,matchstring)) {
				print $1 >> dstfile
				start_found=1
			}
		}
	}'
}

diffreport_print_table_header () {
	local kernel1=$1
	local kernel2=$2

	printf "%-24s%-10s%-10s%-10s\n" "" "$kernel1" "" "$kernel2"

	printf "%-10s%-4s%-4s%-12s%-12s%-12s%-12s%5s%5s\n" "workload" "Set" "NR" "RDBW(KB/s)" "WRBW(KB/s)" "RDBW(KB/s)" "WRBW(KB/s)" "%Rd" "%Wr"
	printf "%-10s%-4s%-4s%-12s%-12s%-12s%-12s%5s%5s\n" "--------" "---" "--" "----------" "----------" "----------" "----------" "----" "----"
}

# Prints the line to output.
diffreport_print_one_line () {
	local workload=$1
	local nrset=$2
	local nrthread=$3
	local readbw1=$4
	local writebw1=$5
	local readbw2=$6
	local writebw2=$7
	local readdiff=$8
	local writediff=$9

	printf "%-10s%-4s%-4s%-12s%-12s%-12s%-12s%5s%5s\n" "$workload" "$nrset" "$nrthread" "$readbw1" "$writebw1" "$readbw2" "$writebw2" "$readdiff%" "$writediff%"
}

diffreport_extract_print_various_test_params () {
	local rawfile=$1

	# Host
	local hostname=`grep "Host=" $rawfile | head -1 | awk '{print $1}' | cut -d "=" -f2`

	# Device
	local device=`grep "DEV=" $rawfile | head -1 | awk '{print $2}' | cut -d "=" -f2`

	# iosched
	local iosched=`grep "iosched=" $rawfile | head -1 | awk '{print $2}' | cut -d "=" -f2`

	# file size
	local filesize=`grep "Filesz=" $rawfile | head -1 | awk '{print $3}' | cut -d "=" -f2`

	# block size
	# Grepping for bs= greps other lines from riostest output hence
	# grepping for Filesz=
	local blocksize=`grep "Filesz=" $rawfile | head -1 | awk '{print $4}' | cut -d "=" -f2`

	echo "Various Test Parameters"
	echo "-----------------------"
	printf "%-30s%-30s\n" "Host=$hostname" "DEV=$device"
	printf "%-16s%-12s%-8s\n" "iosched=$iosched" "Filesz=$filesize" "bs=$blocksize"
}

diffreport_print_misc_info () {
	local rawfile=$1

	# Print what all workloads mean.
	echo "Following are the definitions of various workloads"
	echo "--------------------------------------------------"
	list_available_workloads

	echo
	# Extract and print various test parameters
	diffreport_extract_print_various_test_params $rawfile
	echo
}

# Retrieve either read bw or write bw field from raw data file of workload
# for a specific number of thread and for specific job in workload.
diffreport_get_bw_for_job_nr_thread () {
	local getread=$1
	local job=$2
	local nrthread=$3
	local rawfile=$4

	cat $rawfile | awk -v getread=$getread -v job=$job -v nrthread=$nrthread '{
		if ($1==job && $3==nrthread) {
			if (getread)
				print $4
			else
				print $6
		}
	}'
}

# Retrieve either read bw or write bw field from raw data file of workload
# for a specific number of thread and for specific job in workload.
diffreport_get_bw_for_workload_nr_thread () {
	local getread=$1
	local workload=$2
	local nrthread=$3
	local rawfile=$4
	local aggr=0
	local temp=0

	# Loop through all the components jobs of a workload.
	for job in `component_jobs_of_workload $workload`
	do
		temp=`diffreport_get_bw_for_job_nr_thread "$getread" "$job" "$nrthread" "$rawfile"`
		temp=`printf "%.0f" $temp`
		let aggr=$aggr+$temp
	done
	echo $aggr
}

# Retrieve the max number of nr threads a specific job was run for from a
# rawfile.
diffreport_get_nrthread_for_job () {
	local job=$1
	local rawfile=$2

	cat $rawfile | awk -v job=$job '{
		if ($1==job) {
			maxnrthread=$3
		}
	} END {print maxnrthread}'
}

# max number of threads for a workload.
diffreport_get_nrthread_for_workload () {
	local worklaod=$1
	local rawfile=$2

	for job in `component_jobs_of_workload $workload`
	do
		# Retrieve the max nrthreads for first component job and
		# return
		echo `diffreport_get_nrthread_for_job "$job" "$rawfile"`
		return
	done
}

diffreport_get_nrset_for_job () {
	local job=$1
	local rawfile=$2

	cat $rawfile | awk -v job=$job '{
		if ($1==job) {
			nrset=$2
		}
	} END {print nrset}'
}

diffreport_get_nrset_for_workload () {
	local worklaod=$1
	local rawfile=$2

	for job in `component_jobs_of_workload $workload`
	do
		# Retrieve the max nrthreads for first component job and
		# return
		echo `diffreport_get_nrset_for_job "$job" "$rawfile"`
		return
	done
}

diffreport_calc_percent_diff_of_two_values () {
        local val1=$1
        local val2=$2
        local diff=0

	# Round of decimal input to nearest integer.
	val1=`printf "%.0f" $val1`
	val2=`printf "%.0f" $val2`

        let diff=$val2-$val1
	[ $diff -eq 0 ] && echo 0 && return
        let diff=$diff*100
        let diff=$diff/$val1
        echo $diff
}

# Processes a workload
diffreport_process_job () {
	local job=$1
	local rawfile1=$2
	local rawfile2=$3
	local i=0
	local maxnrthreads=`diffreport_get_nrthread_for_workload "$job" "$rawfile1"`
	local nrset=`diffreport_get_nrset_for_workload "$job" "$rawfile1"`

	for((i=1;i<=$maxnrthreads;i=i*2));do
		readbw1=`diffreport_get_bw_for_workload_nr_thread "1" "$job" "$i" "$rawfile1"`
		readbw2=`diffreport_get_bw_for_workload_nr_thread "1" "$job" "$i" "$rawfile2"`
		writebw1=`diffreport_get_bw_for_workload_nr_thread "0" "$job" "$i" "$rawfile1"`
		writebw2=`diffreport_get_bw_for_workload_nr_thread "0" "$job" "$i" "$rawfile2"`
		readbwdiff=`diffreport_calc_percent_diff_of_two_values "$readbw1" "$readbw2"`
		writebwdiff=`diffreport_calc_percent_diff_of_two_values "$writebw1" "$writebw2"`
		# Print the processed result
		diffreport_print_one_line "$job" "$nrset" "$i" "$readbw1" "$writebw1" "$readbw2" "$writebw2" "$readbwdiff" "$writebwdiff"
	done
}

diffreport_process_workload_stats () {
	local rawfile1=$1
	local rawfile2=$2
	local workload=$3

	local temprawfile1=`mktemp /tmp/iostest.tmp.XXXXX`
	local temprawfile2=`mktemp /tmp/iostest.tmp.XXXXX`

	# Extract AVERAGE data for workload from rawfile to temprawfile
	diffreport_extract_workload_raw_data "$rawfile1" "$temprawfile1" "$workload"
	diffreport_extract_workload_raw_data "$rawfile2" "$temprawfile2" "$workload"

	diffreport_print_table_header "$kernel1" "$kernel2"

	diffreport_process_job "$workload" "$temprawfile1" "$temprawfile2"

	# Echo extra line after each workload
	echo

	# remove tempfiles
	rm $temprawfile1 $temprawfile2
}


# Takes input as two files. These two files are reports of two runs of iostest.
# This function tries to generate a master report where it compares the results
# of two runs and outputs the results.
generate_diff_report () {
	local rawfile1=$1
	local rawfile2=$2
	local tempfile=`mktemp /tmp/iostest.tmp.XXXXX`
	local tempworkloads=`mktemp /tmp/iostest.tmp.XXXXX`

	# Print various test parameters and workload definitions.
	diffreport_print_misc_info "$rawfile1"

	# Determine hostname
	local hostname=`cat $rawfile1 | grep "Host=" | head -1 | awk '{print $1}' | cut -d "=" -f2`

	# Determine kernel1 and kernel2
	local kernel1=`cat $rawfile1 | grep "Kernel=" | head -1 | awk '{print $2}' | cut -d "=" -f2`
	local kernel2=`cat $rawfile2 | grep "Kernel=" | head -1 | awk '{print $2}' | cut -d "=" -f2`

	diffreport_prepare_list_of_workloads "$rawfile1" "$rawfile2" "$tempworkloads"
	# Store list of workloads in an array
	local workloads_arr=( $(cat "$tempworkloads") )

#	echo "List of workload array is"
#	for item in ${workloads_arr[*]}
#	do
#    		printf "%s\n" $item
#	done

	# For each workload, extract READBW
	for item in ${workloads_arr[*]}
	do
		diffreport_process_workload_stats "$rawfile1" "$rawfile2" "$item"
	done
}

print_initial_info () {
	# Print device and TESTDIR info

	printf "%-40s%-30s\n" "DIR=$TESTDIR" "DEV=$BLOCKDEV"

	if [ -n "$GROUPMODE" ];then
		printf "%-40s%-30s\n" "GROUPMODE=1" "NRGRP=$NRGRP"
	fi

	if [ -n "$WORKLOADS" ] || [ -n "$HETROGENOUS_GROUPS" ];then
		true
	else
		echo "No specific workload. Will run all the defined workloads"
	fi

	if [ -n "$NRPROCS" ];then
		true
	else
		echo "Will run workloads for increasing number of threads upto a max of $MAXTHREADS"
	fi
}

# This function decides the major minor number of device. If a partition
# is the blockdev, then we will try to determine minor of whole device.
determine_blockdev_major_minor () {
	local blockdev=$1
	local major
	local minor
	local part
	local baseblockdev=`basename $blockdev`

	major=`cat /proc/partitions | grep -w $baseblockdev | awk '{print $1}'`
	minor=`cat /proc/partitions | grep -w $baseblockdev | awk '{print $2}'`

	if [ ! -d "/sys/dev/block/$major:$minor" ];then
		echo "Error: Dir /sys/dev/block/$major:$minor does not exist"
		exit 1
	fi

	if [ -f "/sys/dev/block/$major:$minor/partition" ];then
		# This looks like a minor number for partition.
		part=`cat /sys/dev/block/$major:$minor/partition`
		# get rid of partition info from blockdev

		# Note double quotes around sed allows it to treat $part
		# as shell variable.
		# cciss partitions also have format p1, p2 etc
		local is_cciss=`echo $blockdev | grep "cciss" | wc -l`
		if [ "$is_cciss" == 1 ];then
			blockdev=`echo $blockdev | sed "s/p$part$//"`
		else
			blockdev=`echo $blockdev | sed "s/$part$//"`
		fi

		baseblockdev=`basename $blockdev`
		major=`cat /proc/partitions | grep -w $baseblockdev | awk '{print $1}'`
		minor=`cat /proc/partitions | grep -w $baseblockdev | awk '{print $2}'`
		if [ ! -d "/sys/dev/block/$major:$minor" ];then
			echo "Error: Dir /sys/dev/block/$major:$minor does not exist"
			exit 1
		fi
	fi

	BLOCKDEVMAJOR=$major
	BLOCKDEVMINOR=$minor
}

# Go through BLOCKDEV_LIST and change ioscheduler on all of these.
change_ioscheduler () {
	local iosched=$1
	local device

	for device in $BLOCKDEV_LIST;do
		[ -n "$DEBUG" ] && echo "Changing iosched to $iosched on device $device"
		echo $iosched > /sys/dev/block/$device/queue/scheduler
	done
}

# Goes through BLOCKDEV_LIST and sets the ioscheduler parameter on all
# devices
set_iosched_parameter () {
	local parameter=$1
	local paramval=$2
	local device

	for device in $BLOCKDEV_LIST;do
		if [ -f "/sys/dev/block/$device/queue/iosched/$parameter" ];then
			[ -n "$DEBUG" ] && echo "Setting $parameter to $paramval on device $device"
			echo $paramval > /sys/dev/block/$device/queue/iosched/$parameter
		fi
	done
}

# Traverse dm tree and add all child devices to BLOCKDEV_LIST.
# I am using a really hackish method of parsing dmsetup ls --tree. If there
# is a better way, please let me know or send a patch.

traverse_dm_tree () {
	local major=$1
	local minor=$2

	# deterine starting line number of device in dmsetup ls --tree output
	local start_line=`dmsetup ls --tree | grep -n "($major:$minor)" | head -1 | awk -F : '{print $1}'`
	if [ "$start_line" == "" ];then
		echo "Error: Can not find ($major:$minor) in dm device tree"
		exit 1
	fi

	local end_line=`dmsetup ls --tree | grep -n -e "^[a-z]" -e "($major:$minor)" | grep -A 1 "($major:$minor)" | head -2 | tail -1 | awk -F : '{print $1}'`

	if [ "$end_line" == "$start_line" ];then
		# This probably is last top level node in the output. So
		# last line is end line of the top level node.
		end_line=`dmsetup ls --tree | wc -l`
	else
		# End line will be 1 less as this number represents the line
		# number of next top level node.
		let end_line=$end_line-1
	fi

	# Extract text between those lines
	BLOCKDEV_LIST=`dmsetup ls --tree | sed -n "$start_line,$end_line p" | sed 's/.*(//g' | sed 's/).*//g' | tr "\\n" " "`
}

# Prepares list of dm child devices and appends in BLOCKDEV_LIST 
prepare_list_of_child_devices () {
	local major=$1
	local minor=$2

	BLOCKDEV_LIST=$major:$minor

	local count=`cat /proc/devices | grep -w "device-mapper"`
	if [ "$count" == "" ];then
		# No device mapper devices
		return
	fi

	local dm_major=`cat /proc/devices | grep -w "device-mapper" | awk '{print $1}'`
	if [ "$dm_major" == "$major" ];then
		# It is a device mapper device. Try to look for children
		# devices and add to BLOCKDEV_LIST.
		traverse_dm_tree $major $minor
	fi	
}

set_defaults () {
	if [ -z "$IOSCHED" ];then
		# By default ioscheduler is cfq
		IOSCHED=cfq
	fi

	if [ -z "$MAXTHREADS" ];then
		# By default max threads is 16. This is used when user has not
		# specified a fixed number of threads to run and we run
		# increasing number of jobs 1, 2, 4, 8,...MAXTHREADS.
		MAXTHREADS=16
	fi

	# By default run 1 set
	[ -z "$NRSETS" ] && NRSETS=1

	if [ -z "$FILESIZE" ];then
		# default file size is 1G. For group mode default file size is
		# 512M. Because nrjobs multiply with number of groups, keeping
		# filesize low by default helps.
		if [ -n "$GROUPMODE" ];then
			FILESIZE=512M
		else
			FILESIZE=1G
		fi
	fi

	if [ -z "$BLOCKSIZE" ];then
		# default block size is 4K
		BLOCKSIZE=4K
	fi

	if [ -z "$RUNTIME" ];then
		# default run time is 30seconds
		RUNTIME=30
	fi

	# Cgroup options.
	# In hetrogenous mode, nrgroups is determined by number of groups
	# specified on command line by option -g
	if [ -n "$HETROGENOUS_GROUPS" ];then
		NRGRP=${#GROUP_DATA_ARRAY[@]}
	elif [ -z "$NRGRP" ];then
		# default number of cgroups to create is 8
		NRGRP=2
	fi

	if [ -z "$GRPISOLATION" ];then
		# grp isolation is enabled by default
		GRPISOLATION=1
	fi

	if [ -z "$SLICE_IDLE" ];then
		# By default slice_idle is 8
		SLICE_IDLE=8
	fi

	if [ -z "$GROUP_IDLE" ];then
		# By default group_idle is 8
		GROUP_IDLE=8
	fi
}

process_input_group_data () {
	local grpdata=$1
	local nrfields=`echo $grpdata | awk -F "#" '{print NF}'`
	local pigd_result

	if [ $nrfields -lt 3 ]; then
		echo "Group string $grpdata is incomplete"
		exit 1
	fi

	local group_name=`echo $grpdata | awk -F "#" '{print $1}'`

	# Currently limit group name to max of 6 characters
	local group_name_len=`echo ${#group_name}`
	if [ $group_name_len -gt 6 ];then
		echo "Group name $group_name is more than 6 char long"
		exit 1
	fi

	local group_weight=`echo $grpdata | awk -F "#" '{print $2}'`
	# Weight has to be with-in 100 to 1000
	if [ $group_weight -lt 100 ] || [ $group_weight -gt 1000 ];then
		echo "Group weight should be between 100 to 1000"
		exit 1
	fi

	local wl=`echo $grpdata | awk -F "#" '{print $3}'`

	verify_and_prepare_workload_string "$wl" pigd_result
	[ $? -ne 0 ] && echo "Bad workload $wl" && exit 1

	# Input group data seems to be fine. Add it to the group array
	GROUP_DATA_ARRAY[$group_name]="$group_weight""#""$pigd_result"
	[ -n "$DEBUG" ] && echo "Added group data [$group_name]=${GROUP_DATA_ARRAY[$group_name]}"
}

run_hetrogenous_workload () {
	# We need to come up with a name for the whole job which can be
	# used as "workload" everywhere. For the time being use a static
	# string "hetro"
	local workload="hetro"
	run_workload "$workload"
}


# Due to usage of bash associative arrays (which are available in bash 4.0 or up# only), check we have right version of bash.

check_bash_version () {
	local maj_version=`bash --version | head -1 | awk '{print $4}' | cut -d "." -f1`
	[ "$maj_version" -lt "4" ] && echo "Bash 4 or higher is needed to run iostest" && exit 1
}

Usage () {
	echo "Usage: $0 [OPTION]... DEVICE"
	echo "Usage: $0 [OPTION]... DIR"
	echo "Usage: $0 -l	: List type of workloads"
	echo "Usage: $0 -R DIR	: Generate report. DIR contains raw data files"
	echo "Usage: $0 -R file1 file2	: Generate master report to compare two runs"
	
	echo "Options:"

	echo "-w  --workload=WL,WL,..	Run specific workloads. Use -l to list workloads. WL can"
	echo "			be either statically defined workload or a custom one."
	echo "			For custom workload use format <custom-wl-str>:WL:WL..."
	echo "			Should work with -R option also to print specific report"
	echo "-n  --nrjobs=N		Launch N number of jobs/processes"
	echo "-m  --maxthreads=N	Do not cross N while testing increasing number of"
	echo "			threads. Default is 16"
	echo "-N  --nrsets=N		Number of sets to run. Default is 1."
	echo "-D  --directory=DIR	DIR where fio scripts and result files are stored."
	echo "			Default is ./iosched-tests/"
	echo "-i  --iosched=STRING	Use STRING ioscheduler for tests"
	echo "-I  --slice_idle=N	Use N for slice_idle value (cfq only)"
	echo "-s  --filesize=SIZE	Use files of size SIZE for tests. Default is 1G"
	echo "    --bs=SIZE		Use SIZE for blocksize. Deafult is 32K"
	echo "-t  --runtime=N		Run workload for N seconds. Default is 30 seconds"
	echo "-q  --quantum=N		Set cfq quantum=N"
	echo "-T  --blktrace=[DEV]	Capture block trace data on DEV. Default is DEVICE on"
	echo "			which iostest is running"
	echo "-c  --cleanup		Remove any *.job, *.txt.bak, *.txt filesin output dir"
	echo "-d  --debug		Debug mode. Outputs extra messages"

	printf "\nCgroup options:\n"
	echo "-G			Group mode. Run jobs in cgroups"
	echo "    --nrgrp=N		Create N cgroups. Default is 2"
	echo "-g  --group=<cgroup-name>#<weight>#<WL>	Create a cgroup with
specified name, weight and workload"
	echo "-r  --rootgrp		Run all group jobs in root group"
	echo "    --grpisolation=<0/1>	Set group isolation. Default is 1."
	echo "-L  --group_idle=N	Set group_idle = N. Default is 8"

	printf "\nReporting options:\n"
	echo "    --cgtime		Report cgroup time information in report. Default is no."
	echo "    --setdata		Print set data. By default only average data is printed"
	echo "    --mergerw		Merge Read/Write stat into one column. Useful for graphs"
	echo "    --noheader		Do not print usual report headers."
	echo "    --total		Print a total BW column for groups."
	echo "-h			Help"
}

# Main Script
# Make sure bash version of 4 or higher
check_bash_version

if [ $# -lt 1 ];then
	Usage
	exit 1
fi

# Note :: indicates optional arguments

args=`getopt -l nrjobs:,filesize:,blktrace::,bs:,iosched:,workload:,maxthreads:,runtime:,nrsets:,directory:,cleanup,debug,nrgrp:,cgtime,setdata,mergerw,noheader,total,group:,rootgrp,grpisolation:,slice_idle:group_idle:quantum: -- w:n:m:s:i:lt:T::hRN:D:cdGrg:I:L:q: $*`
[ $? -ne 0 ] && echo "Error parsing arguments" && exit 1
eval set -- "$args"

while true ; do
        case "$1" in
                -l) LISTWORKLOADS=1; shift 1;;
                -R) GENREPORT=1
		    shift 1
		    ;;
                -G) GROUPMODE=1
		    shift 1
		    ;;
		-w | --workload) WORKLOADS=$2;
			shift 2;;
                -n | --nrjobs) NRPROCS=$2; shift 2;;
		-m | --maxthreads) MAXTHREADS=$2;shift 2;;
		-N | --nrsets) NRSETS=$2;shift 2;;
		-D | --directory) OUTPUTDIR=$2;
				shift 2
				;;
		-i | --iosched) IOSCHED=$2;shift 2;;
		-I | --slice_idle) SLICE_IDLE=$2;shift 2;;
                -s | --filesize) FILESIZE=$2; shift 2;;
		--bs) BLOCKSIZE=$2;shift 2;;
		-t | --runtime) RUNTIME=$2;shift 2;;
		-q | --quantum) QUANTUM=$2;shift 2;;
		-T | --blktrace) BLKTRACE=1;
				 BLKTRACEDEV=$2
				 shift 2
				 ;;
		-c | --cleanup) CLEANUP=1; shift 1;;
		-d | --debug) DEBUG=1; shift 1;;
		# cgroup options
                --nrgrp) NRGRP=$2;
			[ -n "$HETROGENOUS_GROUPS" ] && echo "--nrgrp is not allowed with -g option" && exit 1;
			[ $NRGRP -gt $MAXNRGRP ] && echo "Max number of groups allowed is $MAXNRGRP" && exit 1;
			shift 2;;
                --cgtime) CGTIME=1
		    shift 1
		    ;;
                --setdata) SETDATA=1
		    shift 1
		    ;;

                --mergerw) MERGERW=1
		    shift 1
		    ;;
                --noheader) NOHEADER=1
		    shift 1
		    ;;
                --total) TOTAL=1
		    shift 1
		    ;;
                -r|--rootgrp) ROOTGRP=1
		    shift 1
		    ;;
                -g|--group)
		    # We have hetrogenous group and we have per group data
		    # instead of single set of data applicable to all groups
		    HETROGENOUS_GROUPS=1
		    # group mode is implied with -g
		    GROUPMODE=1
		    process_input_group_data $2
		    shift 2
		    ;;
		--grpisolation) GRPISOLATION=$2;shift 2;;
		-L | --group_idle) GROUP_IDLE=$2;shift 2;;
		-h)
			Usage
			exit 1
			;;
                --) shift ; break ;;
                *) echo "Internal error!" ; exit 1 ;;
        esac
done

if [ "$GENREPORT" == 1 ];then
	# iostest reporting mode.

	# If first argument is DIR, then generate report from raw files.
	# Otherwise if two files have been provided as input, generate
	# diff report of these two files.

	if [ -d "$1" ];then
		RAWFILEDIR=$1
	elif [ -f "$1" ] && [ -f "$2" ];then
		GENDIFFREPORT=1
		RAWREPORTFILE1=$1
		RAWREPORTFILE2=$2
	else
		echo "Either provide dir of raw files or report files to compare"
		exit 1
	fi
else
	# iostest test mode
	# See if first argument is a block special file
	if [ -b "$1" ]; then
		BLOCKDEV=$1
		determine_blockdev_major_minor $BLOCKDEV
		prepare_list_of_child_devices $BLOCKDEVMAJOR $BLOCKDEVMINOR
	fi

	# If it is a directory, user has already mounted the device and passed
	# in a file system dir. Do the testing inside that dir. Set FSMODE=1
	# which indicates that we are running in user specified dir and not a
	# user specified
	# device which we mounted on fixed mount point.
	[ -d "$1" ] && TESTDIR=$1/fio && FSMODE=1
fi

if [ -n "$LISTWORKLOADS" ];then
	list_available_workloads
	exit 1
fi

# might use this data to print report only for specific workloads.
if [ -n "$WORKLOADS" ];then
	# Verify that workload is supported.
	verify_and_prepare_workload_string "$WORKLOADS" RUN_WORKLOADS
	[ $? -ne 0 ] && echo "Bad workload $WORKLOADS" && exit 1
	[ -n "$DEBUG" ] && echo "RUN_WORKLOADS is $RUN_WORKLOADS"
fi

if [ -n "$GENREPORT" ];then
	if [ -n "$GENDIFFREPORT" ];then
		generate_diff_report "$RAWREPORTFILE1" "$RAWREPORTFILE2"
	else
		generate_report "$RAWFILEDIR"
	fi
	exit 1
fi

if [ -z "$BLOCKDEV" ] && [ -z "$FSMODE" ];then
	echo "Enter a valid block dev or a dir"
	exit 1
fi

# Get rid of last number (assuming it is partition number) to come up with
# device name for blktrace

# If user has specified blktrace dev on commnad line, use that otherwise
# try to come up with one based on device on which iostest is running.
if [ -z "$BLKTRACEDEV" ];then
	BLKTRACEDEV=`echo $BLOCKDEV | sed 's/[0-9]$//'`
fi

set_defaults
misc_initializations
sync
echo 3 > /proc/sys/vm/drop_caches

print_initial_info
start_blktrace

if [ -n "$HETROGENOUS_GROUPS" ];then
	run_hetrogenous_workload
elif [ -n "$RUN_WORKLOADS" ];then
	# Run a user specified workloads
	for item in $RUN_WORKLOADS;do
		run_workload "$item"
	done
else
	# Run a string of default workloads
	run_workload "bsr"
	run_workload "brr"
	run_workload "brrmmap"
	run_workload "bufw"
	run_workload "bufwfs"
	run_workload "bufwfs32"
	run_workload "osyncw"
	run_workload "drr"
	run_workload "drw"
	run_workload "drrmmap"
	run_workload "arr"
	run_workload "arw"
#	run_workload "mixed"
	run_workload "database"
	run_workload "desktop"
#	run_workload "kvmhost"
fi

stop_blktrace
generate_report "$OUTPUTDIR"
cleanup_before_exit
