Merge pull request #66 from redhat-performance/handle_all_tests

Updates to handle all the various file results formats that we have.

Author: dvalinrh

validate_line

@@ -19,20 +19,27 @@
 
 exit_rtc=0
 base_results_file=""
+header=""
+compare_file=`mktemp /tmp/validate_compare.XXXXX`
+verification_base=`mktemp /tmp/validate_veri_base.XXXXX`
+verify_template=`mktemp /tmp/validate_veri_temp.XXXXX`
+test_results=`mktemp /tmp/validate_test_res.XXXXX`
 
 usage()
 {
 	echo Usage: $1
-	echo "  --fields <string>: Contains a list in the following format"
-	echo "       name:type,name:type"
-	echo "    Valid types are"
-	echo "     n: field is a numeric value"
-	echo "     s: field is a string of some sort"
-	echo "     sm: field is a string and must match exactly the name"
 	echo "  --header_lines <n>: How many lines of header do we we expect."
 	echo "  --results_file <path>: The results file generated by the test"
 	echo "  --base_results_file <path>:  The path to the expected results that we are"
 	echo "     to compare to."
+	echo File format
+	echo "  Format of file is expected to be csv with ':' as field separators"
+	echo "  Directives:"
+	echo "    %_header: Next line is the header"
+	echo "    %_passes: We have multiple passes of the test.  First column in the csv"
+	echo "       file will be the pass number"
+	echo "    %_multiples:  We have multiple lines of the data."
+	echo "    %_grouping: The data will appear in groups."
 	exit $2
 }
 
@@ -56,6 +63,9 @@ validate_lines()
 	do
 		regexp=`echo "$test_info" |  cut  -f 1`
 		string=`echo "$test_info" |  cut  -f 2`
+		if [[ $string == "" ]]; then
+			continue
+		fi
 		#
 		# Check to see if the string matches the regular expression.
 		#
@@ -68,7 +78,6 @@ validate_lines()
 }
 
 ARGUMENT_LIST=(
-	"fields"
 	"header_lines"
 	"results_file"
 	"base_results_file"
@@ -88,7 +97,6 @@ opts=$(getopt \
 eval set --$opts
 
 results_file=""
-fields=""
 header_lines=0
 
 while [[ $# -gt 0 ]]; do
@@ -97,10 +105,6 @@ while [[ $# -gt 0 ]]; do
 			base_results_file=$2
 			shift 2
 		;;
-		--fields)
-			fields=`echo $2 | sed "s/ /_/g" | sed "s/,/ /g"`
-			shift 2
-		;;
 		--header_lines)
 			header_lines=$2
 			shift 2
@@ -125,13 +129,73 @@ while [[ $# -gt 0 ]]; do
 	esac
 done
 
-meta_head=0
 meta_test=0
 header_line=""
 field_location=""
 separ=""
 field_index=1
-rm -f tmp_results compare_file
+
+#
+# First create the verification base file.
+#
+iteration_test=0
+lines_per_pass=0
+multiple_entries=0
+multiple_groups=0
+multiple_search=""
+header_next=0
+grouping=0
+
+while IFS= read -r verif_info
+do
+	if [[ $header_next -eq 1 ]]; then
+		echo $verif_info >> $verification_base
+		header_next=0
+		continue
+	fi
+	if [[ $verif_info == *"%_passes"* ]]; then
+		iteration_test=1
+		continue
+	fi
+	if [[ $verif_info == *"%_multiples"* ]]; then
+		multiple_entries=1
+		continue
+	fi
+	if [[ $verif_info == *"%_header"* ]]; then
+		header_next=1
+		continue
+	fi
+	if [[ $verif_info == *"%_grouping"* ]]; then
+		grouping=1
+		continue
+	fi
+	if [[ $multiple_entries -eq 1 ]] || [[ $grouping -eq 1 ]]; then
+		echo $verif_info >> $verification_base
+		continue
+	fi
+	if [[ $iteration_test -eq 1 ]]; then
+		if  [[ $verif_info != *"iteration"* ]]; then
+			let "lines_per_pass=$lines_per_pass+1"
+			#
+			# Iteration number will be added later.
+			#
+			output=`echo $verif_info | cut -d: -f2-`
+			echo $output >> $verification_base
+		else 
+			header=$verif_info
+		fi
+	else
+		echo $verif_info >> $verification_base
+	fi
+done < "$base_results_file"
+
+lines=0
+iterations=1
+header_processed=0
+
+grep -q "Test general meta end" $results_file
+meta_head=$?
+echo meta_head $meta_head
 while IFS= read -r test_info
 do
 	if [ $meta_head -eq 0 ]; then
@@ -143,6 +207,7 @@ do
 		fi
 		continue
 	fi
+
 	if [[ $test_info == *"Test meta data start"* ]]; then
 		#
 		# Skip over the test meta data.
@@ -151,18 +216,68 @@ do
 		continue
 	fi
 	if [[ $meta_test -eq 1 ]]; then
-		if [[ $test_info == *"Test meta data end*" ]]; then
+		if [[ $test_info == *"Test meta data end"* ]]; then
 			meta_test=0
 		fi
 		continue
 	fi
-	echo $test_info >> tmp_results
+	if [[ $iteration_test -eq 1 ]]; then
+		base_results_file=$verify_template
+		let "lines=$lines+1"
+		if [[ $lines -eq $lines_per_pass ]]; then
+			while IFS= read -r verf_info
+			do
+				if [[ $verf_info == *"iteration"* ]]; then
+					continue
+				else
+					echo $iterations:$verf_info >> $verify_template
+				fi
+			done < $verification_base
+			let "iterations=$iterations+1"
+			lines=0
+		fi 
+	fi
+	if [[ $grouping -eq 1 ]] && [[ $groups_done -eq 0 ]]; then
+		groups_done=1
+		field=`echo $test_info | cut -d: -f 1`
+		inst_cnt=`grep "^${field}:" $results_file | wc -l`
+		let "inst_cnt=$inst_cnt+1"
+		for i in `seq 0 $inst_cnt`;do
+			cat $verification_base >> $verify_template
+		done
+		base_results_file=$verify_template
+	fi
+	if [[ $multiple_entries -eq 1 ]]; then
+		next_line_header=0
+		base_results_file=$verify_template
+		line=0
+		while IFS= read -r verf_info
+		do
+			if [[  $line -eq 0 ]]; then
+				if [[ $header_processed -eq 0 ]]; then
+					echo "$verf_info" >> $verify_template
+					header_processed=1
+					break
+				fi
+				let "line=$line+1"
+			else
+				echo "$verf_info" >> $verify_template
+			fi
+		done < $verification_base
+		header_processed=1
+	fi
+	echo $test_info >> $test_results
 done < "$results_file"
 #
 # Create the compare file  by pasting the base results (reg exp) against the tmp_results
 # file which is minus all the meta documentation.
 #
-paste $base_results_file tmp_results > compare_file
-validate_lines compare_file
-rm compare_file
+paste $base_results_file $test_results > $compare_file
+validate_lines $compare_file
+rm -f $compare_file $verification_base $verify_template $test_results
+if [[ $exit_rtc -ne 0 ]]; then
+	echo Failed
+else
+	echo Ran
+fi
 exit $exit_rtc

verification

@@ -36,6 +36,10 @@ verification_run()
 	verify_res_before=`mktemp ${tmp_dir}/wrapper_verify.XXXXX`
 	verify_res_after=`mktemp ${tmp_dir}wrapper_verify.XXXXX`
 	test_descript=$verify_file
+	if [[ ! -f $test_descript ]]; then
+		echo $test_descript does not exist.
+		exit 1
+	fi
 	required_systems=`grep "^Required_Systems:" $test_descript | cut -d: -f2`
 	fields=`grep ^fields $test_descript | cut -d: -f2-`
 	header_lines=`grep ^header_lines $test_descript | cut -d: -f2`
@@ -63,10 +67,11 @@ verification_run()
 		rdir=`diff $verify_res_after $verify_res_before | grep -v tar | grep '<' |  sed "s/< //g"`
 		res_dir=${home_parent}/${run_user}/export_results/$rdir
 		#
-		# Locate the test csv file
+		# Locate the test csv file.  First grep for results_{$test_name} then grep for the csv, to
+		# keep the tests that have sub tests (hammer) happy.
 		#
-		new_res_file=`find $res_dir -print | grep results_${test_name}.csv`
-		$TOOLS_BIN/validate_line --results_file $new_res_file --fields "$fields" --base_results_file $base_res_file
+		new_res_file=`find $res_dir -print | grep results_${test_name} | grep csv`
+		$TOOLS_BIN/validate_line --results_file $new_res_file --base_results_file $base_res_file
 		if [[ $? -ne 0 ]]; then
 			exit_rtc=1
 		fi