From 19d1a015498034d4b3045b1bf43dde96af22f751 Mon Sep 17 00:00:00 2001 From: danieyan-amd Date: Tue, 23 Sep 2025 17:58:06 -0400 Subject: [PATCH 1/2] Add ONNX model testing workflow --- .github/workflows/onnx.yaml | 99 +++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/onnx.yaml diff --git a/.github/workflows/onnx.yaml b/.github/workflows/onnx.yaml new file mode 100644 index 00000000000..154f11b8b2e --- /dev/null +++ b/.github/workflows/onnx.yaml @@ -0,0 +1,99 @@ +name: MIGraphX ONNX Model Tests + +on: + pull_request_target: + branches: [develop] + types: [opened, synchronize, closed] + schedule: + - cron: "0 7 * * 1-6" + workflow_dispatch: + inputs: + rocm_release: + description: ROCm Version + required: true + default: "6.4.2" + onnx_reports_repo: + description: Repository where ONNX reports are stored + required: true + default: "danieyan-amd/migraphx-reports" + benchmark_utils_repo: + description: Repository where benchmark utils are stored + required: true + default: "danieyan-amd/migraphx-benchmark-utils" + organization: + description: Organization based on which location of files will be different + required: true + default: "AMD" + result_number: + description: Last N results + required: true + default: "10" + model_timeout: + description: If a model exceeds this timeout in the test script, it will be skipped + required: true + default: "30m" + +permissions: + contents: write + +concurrency: + group: "perftest-${{ github.head_ref || github.base_ref || 'schedule' }}" + cancel-in-progress: true + +jobs: + get_config: + runs-on: ubuntu-latest + outputs: + rocm_version: ${{ steps.read_config.outputs.rocm_version }} + utils_repo: ${{ steps.read_config.outputs.utils_repo }} + reports_repo: ${{ steps.read_config.outputs.reports_repo }} + repo_org: ${{ steps.read_config.outputs.repo_org }} + perf_number: ${{ steps.read_config.outputs.perf_number }} + perf_timeout: ${{ steps.read_config.outputs.perf_timeout }} + steps: + - name: Checkout (this repo) + uses: actions/checkout@v4 + + - name: Read config defaults + id: read_config + shell: bash + run: | + set -euo pipefail + CONFIG=".github/workflows/config.md" + grab() { + if [[ -f "$CONFIG" ]]; then + (grep -m1 "$1" "$CONFIG" | cut -d"'" -f2) || true + fi + } + ROCM_VERSION="$(grab ROCM_VERSION)"; : "${ROCM_VERSION:=6.4.2}" + BENCHMARK_UTILS_REPO="$(grab BENCHMARK_UTILS_REPO)"; : "${BENCHMARK_UTILS_REPO:=danieyan-amd/migraphx-benchmark-utils}" + ONNX_REPORTS_REPO="$(grab ONNX_REPORTS_REPO)"; : "${ONNX_REPORTS_REPO:=danieyan-amd/migraphx-reports}" + ORGANIZATION_REPO="$(grab ORGANIZATION_REPO)"; : "${ORGANIZATION_REPO:=AMD}" + RESULTS_TO_COMPARE="$(grab RESULTS_TO_COMPARE)"; : "${RESULTS_TO_COMPARE:=10}" + PERFORMANCE_TEST_TIMEOUT="$(grab PERFORMANCE_TEST_TIMEOUT)"; : "${PERFORMANCE_TEST_TIMEOUT:=30m}" + echo "rocm_version=$ROCM_VERSION" >> "$GITHUB_OUTPUT" + echo "utils_repo=$BENCHMARK_UTILS_REPO" >> "$GITHUB_OUTPUT" + echo "reports_repo=$ONNX_REPORTS_REPO" >> "$GITHUB_OUTPUT" + echo "repo_org=$ORGANIZATION_REPO" >> "$GITHUB_OUTPUT" + echo "perf_number=$RESULTS_TO_COMPARE" >> "$GITHUB_OUTPUT" + echo "perf_timeout=$PERFORMANCE_TEST_TIMEOUT">> "$GITHUB_OUTPUT" + echo "Resolved config:" + echo " ROCM_VERSION=$ROCM_VERSION" + echo " BENCHMARK_UTILS_REPO=$BENCHMARK_UTILS_REPO" + echo " ONNX_REPORTS_REPO=$ONNX_REPORTS_REPO" + echo " ORGANIZATION_REPO=$ORGANIZATION_REPO" + echo " RESULTS_TO_COMPARE=$RESULTS_TO_COMPARE" + echo " PERFORMANCE_TEST_TIMEOUT=$PERFORMANCE_TEST_TIMEOUT" + + call_reusable: + needs: get_config + uses: danieyan-amd/migraphx-benchmark/.github/workflows/onnx-model.yaml@main + with: + rocm_release: ${{ github.event.inputs.rocm_release || needs.get_config.outputs.rocm_version }} + benchmark_utils_repo: ${{ github.event.inputs.benchmark_utils_repo || needs.get_config.outputs.utils_repo }} + onnx_reports_repo: ${{ github.event.inputs.onnx_reports_repo || needs.get_config.outputs.reports_repo }} + organization: ${{ github.event.inputs.organization || needs.get_config.outputs.repo_org }} + result_number: ${{ github.event.inputs.result_number || needs.get_config.outputs.perf_number }} + model_timeout: ${{ github.event.inputs.model_timeout || needs.get_config.outputs.perf_timeout }} + secrets: + gh_token: ${{ secrets.MIGRAPHX_BOT_TOKEN }} From 64eef822fe1af5667f6763973252457947bdd88f Mon Sep 17 00:00:00 2001 From: danieyan-amd Date: Tue, 23 Sep 2025 18:42:40 -0400 Subject: [PATCH 2/2] Add script to summarize ONNX Model Zoo test logs This script summarizes MIGraphX ONNX Model Zoo test logs by reading log files, counting passes and failures, and generating summary outputs in JSON and Markdown formats. --- scripts/summarize_onnx_logs.py | 210 +++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 scripts/summarize_onnx_logs.py diff --git a/scripts/summarize_onnx_logs.py b/scripts/summarize_onnx_logs.py new file mode 100644 index 00000000000..2693cfd2d77 --- /dev/null +++ b/scripts/summarize_onnx_logs.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +""" +Summarize MIGraphX ONNX Model Zoo test logs. +Reads *.log files from fp32/ and fp16/ under a results directory, +counts passes/failures, extracts a short failure message, and writes: + - summary.json + - summary.md +Extras: + - --previous: compare to previous run's summary.json (pass-rate delta) + - --write-index: generate index.md with a clickable table of models + - --artifact-url: best-effort link target used in index rows (run page) + - --step-summary: write a compact overview into $GITHUB_STEP_SUMMARY +Usage: + python3 scripts/summarize_onnx_logs.py --results \ + [--out-json ] [--out-md ] \ + [--previous ] [--write-index ] \ + [--artifact-url ] [--step-summary ] +""" +from __future__ import annotations +import argparse +import json +import os +import re +from pathlib import Path + +FAIL_PAT = re.compile( + r"(Traceback \(most recent call last\)|\bERROR\b|AssertionError|Segmentation fault|^error:)", + re.I | re.M, +) +TBLOCK_PAT = re.compile(r"Traceback \(most recent call last\):([\s\S]*?)(?:\n\s*\n|\Z)") + + +def looks_failed(text: str) -> bool: + return bool(FAIL_PAT.search(text)) + + +def failure_message(text: str) -> str: + # last traceback block, last non-empty line + blocks = list(TBLOCK_PAT.finditer(text)) + if blocks: + for line in reversed(blocks[-1].group(1).strip().splitlines()): + line = line.strip() + if line: + return line + # fallback: last interesting line + for line in reversed([l.strip() for l in text.splitlines() if l.strip()]): + if re.search(r"(error|exception|failed|segmentation fault|assert)", line, re.I): + return line + return "failed (see log)" + + +def summarize(results_dir: Path) -> dict: + precs = ("fp32", "fp16") + summary = {"totals": {"pass": 0, "fail": 0}, "regressions": {}} + + for prec in precs: + d = results_dir / prec + reg = {"passed": [], "failed": []} + if d.is_dir(): + files = sorted(p for p in d.glob("*.log")) + for p in files: + model = p.stem + try: + txt = p.read_text(errors="ignore") + except Exception: + txt = "" + if looks_failed(txt): + reg["failed"].append({"model": model, "message": failure_message(txt)}) + summary["totals"]["fail"] += 1 + else: + reg["passed"].append(model) + summary["totals"]["pass"] += 1 + summary["regressions"][prec] = reg + return summary + + +def write_outputs(summary: dict, out_json: Path, out_md: Path) -> None: + out_json.parent.mkdir(parents=True, exist_ok=True) + out_md.parent.mkdir(parents=True, exist_ok=True) + + out_json.write_text(json.dumps(summary, indent=2)) + + lines = [ + "## Totals", + f"- PASS: {summary['totals']['pass']}", + f"- FAIL: {summary['totals']['fail']}", + "", + ] + for prec in ("fp32", "fp16"): + reg = summary["regressions"][prec] + lines.append(f"## {prec.upper()}") + lines.append(f"**Passed ({len(reg['passed'])})**") + if reg["passed"]: + lines.extend([f"- {m}" for m in reg["passed"]]) + else: + lines.append("- none") + lines.append("") + lines.append(f"**Failed ({len(reg['failed'])})**") + if reg["failed"]: + lines.extend([f"- {it['model']}: `{it['message']}`" for it in reg["failed"]]) + else: + lines.append("- none") + lines.append("") + out_md.write_text("\n".join(lines)) + + +def build_index_md(summary: dict, artifact_url: str | None) -> str: + """Return an index.md string with a model table and best-effort links.""" + # Collect models and statuses + models = set() + status: dict[str, dict[str, str]] = {"fp32": {}, "fp16": {}} + for prec in ("fp32", "fp16"): + for m in summary["regressions"][prec]["passed"]: + models.add(m) + status[prec][m] = "pass" + for it in summary["regressions"][prec]["failed"]: + models.add(it["model"]) + status[prec][it["model"]] = "fail" + + def cell(prec: str, model: str) -> str: + st = status[prec].get(model) + if not st: + return "" + emoji = "✅" if st == "pass" else "❌" + rel = f"{prec}/{model}.log" + # If artifact_url is provided, link to the run page and show the relative path below. + # (Direct deep-links to a file in an artifact page are not guaranteed.) + if artifact_url: + return f"[{emoji}]({artifact_url})
{rel}" + else: + # When opening index.md *inside* the artifact, relative links work. + return f"[{emoji}]({rel})" + + rows = ["| Model | FP32 | FP16 |", "|---|:---:|:---:|"] + for model in sorted(models): + rows.append(f"| {model} | {cell('fp32', model)} | {cell('fp16', model)} |") + return "\n".join(rows) + + +def write_step_summary(summary: dict, prev: dict | None, index_hint: str, step_summary_path: Path) -> None: + cur_total = summary["totals"]["pass"] + summary["totals"]["fail"] + cur_rate = (summary["totals"]["pass"] / cur_total) * 100 if cur_total else 0.0 + if prev: + prev_total = prev["totals"]["pass"] + prev["totals"]["fail"] + prev_rate = (prev["totals"]["pass"] / prev_total) * 100 if prev_total else 0.0 + delta = cur_rate - prev_rate + comp = f"**Pass rate:** {cur_rate:.1f}% (Δ {delta:+.1f} pts vs. previous {prev_rate:.1f}%)" + else: + comp = f"**Pass rate:** {cur_rate:.1f}% (no previous run found)" + + failed = [] + for prec in ("fp32", "fp16"): + failed.extend([f"{prec}:{it['model']} — {it['message'][:80]}" for it in summary["regressions"][prec]["failed"]]) + failed_block = "\n".join([f"- {l}" for l in failed[:20]]) or "- none" + + text = [ + "# MIGraphX ONNX Model Zoo — Summary", + comp, + "", + f"Artifacts: {index_hint}", + "", + "## Top failures (first 20)", + failed_block, + ] + # Write (overwrites prior content for this step) + step_summary_path.write_text("\n".join(text)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--results", required=True, help="Results directory containing fp32/fp16 log folders") + ap.add_argument("--out-json", default=None, help="Path to write summary.json (default: /summary.json)") + ap.add_argument("--out-md", default=None, help="Path to write summary.md (default: /summary.md)") + ap.add_argument("--previous", default=None, help="Path to previous summary.json for comparison") + ap.add_argument("--artifact-url", default=None, help="Run or artifact URL to use for links in index.md (best-effort)") + ap.add_argument("--write-index", default=None, help="Write a model table index.md here (optional)") + ap.add_argument("--step-summary", default=None, help="Write a compact overview to this path (e.g., $GITHUB_STEP_SUMMARY)") + args = ap.parse_args() + + results_dir = Path(args.results).expanduser().resolve() + if not results_dir.exists(): + print(f"ERROR: results dir not found: {results_dir}") + return 2 + + summary = summarize(results_dir) + + out_json = Path(args.out_json) if args.out_json else results_dir / "summary.json" + out_md = Path(args.out_md) if args.out_md else results_dir / "summary.md" + write_outputs(summary, out_json, out_md) + + prev = None + if args.previous and Path(args.previous).exists(): + try: + prev = json.loads(Path(args.previous).read_text()) + except Exception: + prev = None + + if args.write_index: + idx = build_index_md(summary, args.artifact_url) + Path(args.write_index).write_text(idx) + + if args.step_summary: + hint = args.artifact_url or "(open run → Artifacts → logs artifact → index.md)" + write_step_summary(summary, prev, hint, Path(args.step_summary)) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())