WIP: Results verification

csv_to_json

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+#
+#                         License
+#
+# Copyright (C) 2025  Keith Valin [email protected]
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# Convert CSV files to JSON.
+
+import argparse
+import sys
+from typing import TextIO
+
+try:
+    import pandas as pd
+except ImportError:
+    print("WARNING: pandas is not installed. Installing using pip", file=sys.stderr)
+    import subprocess
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "--user", "--quiet"])
+    import pandas as pd
+
+def main(csv_file: TextIO, output: TextIO, separator: str):
+    """
+    Convert a CSV file to JSON.
+    """
+    df = pd.read_csv(csv_file, sep=separator, comment='#')
+    df.to_json(output, orient='records', lines=False, indent=4)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Convert CSV files to JSON')
+    parser.add_argument('--csv_file', type=argparse.FileType("r"), help='The CSV file to convert', default=sys.stdin)
+    parser.add_argument('--output_file', type=argparse.FileType("w"), help='The JSON file to write', default=sys.stdout)
+    parser.add_argument('--json_skip', action='store_true', help='Skip processing the CSV to JSON, effectively makes this a no-op')
+
+    # Colon is used as the default separator since most wrappers use it.
+    parser.add_argument('--separator', type=str, help='The separator to use for the CSV file', default=":")
+    args = parser.parse_args()
+
+    if args.json_skip:
+        sys.exit(0)
+
+    main(args.csv_file, args.output_file, args.separator)

general_setup

@@ -76,6 +76,7 @@ to_test_verify_file=""
 to_sys_type=`hostname`
 to_configuration=`hostname`
 to_home_root=`echo $HOME | rev | cut -d'/' -f 2- | rev`
+to_script_dir=`dirname $(realpath $0)`
 if [[ $to_home_root == "" ]]; then
 	to_home_root="/"
 fi
@@ -91,7 +92,8 @@ to_user=`whoami`
 to_sysname=`hostname`
 to_pstats="default"
 to_no_pkg_install=0
-
+to_verify_flags=""
+to_json_flags=""
 to_tuned_setting=""
 to_use_pcp=0
 
@@ -200,6 +202,14 @@ do
 		--usage)
 			gs_usage_info
 		;;
+		--verify_skip)
+			to_verify_flags="$1"
+			shift 1
+		;;
+		--json_skip)
+			to_json_flags="$1"
+			shift 1
+		;;
 		--use_pcp)
  			i=$((i + 1))
  			to_use_pcp=1

verify_results

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+from typing import TextIO
+
+try:
+	from pydantic import BaseModel, TypeAdapter, ValidationError
+except ImportError as e:
+	print("WARNING: Could not import pydantic, installing it using pip", file=sys.stderr)
+	import subprocess
+	subprocess.check_call([sys.executable, "-m", "pip", "install", "pydantic", "--user", "--quiet"])
+	from pydantic import BaseModel, TypeAdapter, ValidationError
+
+def verify_schema(file: TextIO, class_name: BaseModel):
+	data = json.load(file)
+	try:
+		TypeAdapter(list[class_name]).validate_python(data)
+	except ValidationError as e:
+		print("Could not verify schema, see below for details", file=sys.stderr)
+		print(e, file=sys.stderr)
+		sys.exit(1)
+
+	print("Results verified")
+
+if __name__ == "__main__":
+	import argparse
+	import importlib.util
+	import re
+
+	parser = argparse.ArgumentParser()
+	parser.add_argument("--file",
+		type=argparse.FileType("r"),
+		help="JSON file to verify, default is to read from stdin",
+		default=sys.stdin
+	)
+	parser.add_argument("--schema_file",
+		type=str,
+		help="Schema file used to validate JSON file"
+	)
+	parser.add_argument("--class_name", type=str, help="Class name used to validate JSON file", default="Results")
+	parser.add_argument("--usage", action="store_true", help="Show usage")
+	parser.add_argument("--verify_skip", action="store_true", help="Skip verification process")
+	args = parser.parse_args()
+
+	if args.usage:
+		parser.print_help()
+		sys.exit(0)
+
+	if args.verify_skip:
+		sys.exit(0)
+
+	try:
+		# Get the file name (minus the extension), since it is the module name
+		module_name = re.sub(r".py$", "", args.schema_file).split("/")[-1]
+
+		# Import the class from the schema file
+		spec = importlib.util.spec_from_file_location(module_name, args.schema_file)
+		importedClass = importlib.util.module_from_spec(spec)
+		spec.loader.exec_module(importedClass)
+
+		baseModel = getattr(importedClass, args.class_name)
+	# Handle file issues and if the class is not found (attribute error)
+	except (FileNotFoundError, AttributeError) as e:
+		print(f"Class {args.class_name} not found in {args.schema_file}", file=sys.stderr)
+		print(f"Error: {e}", file=sys.stderr)
+		sys.exit(1)
+	except Exception as e:
+		print(f"Error: {e}", file=sys.stderr)
+		sys.exit(1)
+
+	verify_schema(args.file, baseModel)