From 282041685f1a8430498c6cef8f1585d86bdaa0d6 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Fri, 4 Jul 2025 14:18:32 +0100
Subject: [PATCH 01/20] Ensure packages are installed on images

---
 tests/data/modules/data.json        | 260 ++++++++++++++++++++++++++++
 tests/suite/test_docker_packages.py |  18 ++
 2 files changed, 278 insertions(+)
 create mode 100644 tests/data/modules/data.json
 create mode 100644 tests/suite/test_docker_packages.py

diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
new file mode 100644
index 0000000000..9faa657223
--- /dev/null
+++ b/tests/data/modules/data.json
@@ -0,0 +1,260 @@
+{
+  "images": [
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge",
+      "packages": [
+        "nginx",
+        "nginx-module-njs",
+        "nginx-module-otel",
+        "nginx-agent"
+      ],
+      "cmd": "dpkg -l",
+      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "platform": "linux/arm64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent"
+      ],
+      "cmd": "dpkg -l",
+      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "platform": "linux/arm64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-plus-module-appprotect",
+        "nginx-agent"
+      ],
+      "cmd": "dpkg -l",
+      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-plus-module-appprotect",
+        "nginx-agent"
+      ],
+      "cmd": "dpkg -l",
+      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress:edge",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-plus-module-appprotectdos"
+      ],
+      "cmd": "dpkg -l",
+      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress:edge",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-plus-module-appprotect",
+        "nginx-plus-module-appprotectdos",
+        "nginx-agent"
+      ],
+      "cmd": "dpkg -l",
+      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge-alpine",
+      "packages": [
+        "nginx",
+        "nginx-module-njs",
+        "nginx-module-otel",
+        "nginx-agent"
+      ],
+      "cmd": "apk list",
+      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "platform": "linux/arm64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-alpine",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent"
+      ],
+      "cmd": "apk list",
+      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "platform": "linux/arm64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-alpine-fips",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent"
+      ],
+      "cmd": "apk list",
+      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "platform": "linux/arm64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-alpine-fips",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent",
+        "nginx-plus-module-appprotect"
+      ],
+      "cmd": "apk list",
+      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-alpine-fips",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent",
+        "nginx-plus-module-appprotect"
+      ],
+      "cmd": "apk list",
+      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge-ubi",
+      "packages": [
+        "nginx",
+        "nginx-module-njs",
+        "nginx-module-otel",
+        "nginx-agent"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/arm64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-ubi",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/arm64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-ubi",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent",
+        "nginx-plus-module-appprotect"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-ubi",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent",
+        "nginx-plus-module-appprotect"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-ubi8",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent",
+        "nginx-plus-module-appprotect"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-ubi8",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-agent",
+        "nginx-plus-module-appprotect"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress:edge-ubi",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-plus-module-appprotectdos"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/amd64"
+    },
+    {
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress:edge-ubi",
+      "packages": [
+        "nginx-plus",
+        "nginx-plus-module-njs",
+        "nginx-plus-module-otel",
+        "nginx-plus-module-fips-check",
+        "nginx-plus-module-appprotect",
+        "nginx-plus-module-appprotectdos",
+        "nginx-agent"
+      ],
+      "cmd": "rpm -q",
+      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "platform": "linux/amd64"
+    }
+  ]
+}
diff --git a/tests/suite/test_docker_packages.py b/tests/suite/test_docker_packages.py
new file mode 100644
index 0000000000..943714349c
--- /dev/null
+++ b/tests/suite/test_docker_packages.py
@@ -0,0 +1,18 @@
+import json
+import re
+
+import docker
+
+client = docker.from_env()
+images = json.loads(open("tests/data/modules/data.json").read())
+
+for image in images["images"]:
+    regexInstalled = image["regex"]
+    for package in image["packages"]:
+        command = f"{image['cmd']} {package}"
+        output = client.containers.run(
+            image["image"], command, entrypoint="", platform=image["platform"], auto_remove=True, detach=False
+        )
+        result = re.search(regexInstalled, output.decode("utf-8").strip())
+        assert result, f"{package} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
+        print(image["image"], result.group(1, 2, 3))

From 20294fcba84ba0a448723ff8bc2bd1b84ec18e63 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Fri, 4 Jul 2025 15:36:29 +0100
Subject: [PATCH 02/20] Expand to include package versions

---
 tests/data/modules/data.json        | 774 ++++++++++++++++++++++++----
 tests/suite/test_docker_packages.py |   7 +-
 2 files changed, 675 insertions(+), 106 deletions(-)

diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
index 9faa657223..3185a13ad8 100644
--- a/tests/data/modules/data.json
+++ b/tests/data/modules/data.json
@@ -3,10 +3,26 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge",
       "packages": [
-        "nginx",
-        "nginx-module-njs",
-        "nginx-module-otel",
-        "nginx-agent"
+        {
+          "name": "nginx",
+          "version": "1.27.5",
+          "arch": "arm64"
+        },
+        {
+          "name": "nginx-module-njs",
+          "version": "1.27.5",
+          "arch": "arm64"
+        },
+        {
+          "name": "nginx-module-otel",
+          "version": "1.27.5",
+          "arch": "arm64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "3.0",
+          "arch": "arm64"
+        }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
@@ -15,11 +31,31 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "arm64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "arm64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "arm64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "arm64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "3.0",
+          "arch": "arm64"
+        }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
@@ -28,12 +64,51 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-plus-module-appprotect",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect",
+          "version": "34+5.442",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-attack-signatures",
+          "version": "2025",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-threat-campaigns",
+          "version": "2025",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "amd64"
+        }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
@@ -42,12 +117,46 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-plus-module-appprotect",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-module-plus",
+          "version": "34+5.442",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-plugin",
+          "version": "6.16.0",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "amd64"
+        }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
@@ -56,11 +165,36 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress:edge",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-plus-module-appprotectdos"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-appprotectdos",
+          "version": "34+4",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-dos",
+          "version": "34+4",
+          "arch": "amd64"
+        }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
@@ -69,13 +203,61 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress:edge",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-plus-module-appprotect",
-        "nginx-plus-module-appprotectdos",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect",
+          "version": "34+5.442",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-attack-signatures",
+          "version": "2025",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-threat-campaigns",
+          "version": "2025",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-plus-module-appprotectdos",
+          "version": "34+4",
+          "arch": "amd64"
+        },
+        {
+          "name": "app-protect-dos",
+          "version": "34+4",
+          "arch": "amd64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "amd64"
+        }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
@@ -84,10 +266,26 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge-alpine",
       "packages": [
-        "nginx",
-        "nginx-module-njs",
-        "nginx-module-otel",
-        "nginx-agent"
+        {
+          "name": "nginx",
+          "version": "1.27.5",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-module-njs",
+          "version": "1.27.5",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-module-otel",
+          "version": "1.27.5",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "3.0",
+          "arch": "aarch64"
+        }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
@@ -96,11 +294,31 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-alpine",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-r2",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "3.0",
+          "arch": "aarch64"
+        }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
@@ -109,11 +327,31 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-alpine-fips",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-r2",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "3.0",
+          "arch": "aarch64"
+        }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
@@ -122,12 +360,51 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-alpine-fips",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent",
-        "nginx-plus-module-appprotect"
+        {
+          "name": "nginx-plus",
+          "version": "34-r2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34.5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect",
+          "version": "34.5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-attack-signatures",
+          "version": "2025",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-threat-campaigns",
+          "version": "2025",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
@@ -136,12 +413,46 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-alpine-fips",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent",
-        "nginx-plus-module-appprotect"
+        {
+          "name": "nginx-plus",
+          "version": "34-r2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34.5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-module-plus",
+          "version": "34.5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-plugin",
+          "version": "6.16.0",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
@@ -150,10 +461,26 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge-ubi",
       "packages": [
-        "nginx",
-        "nginx-module-njs",
-        "nginx-module-otel",
-        "nginx-agent"
+        {
+          "name": "nginx",
+          "version": "1.27.5",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-module-njs",
+          "version": "1.27.5",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-module-otel",
+          "version": "1.27.5",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "3.0",
+          "arch": "aarch64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
@@ -162,11 +489,31 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-ubi",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "aarch64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "3.0",
+          "arch": "aarch64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
@@ -175,12 +522,51 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-ubi",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent",
-        "nginx-plus-module-appprotect"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-attack-signatures",
+          "version": "2025",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-threat-campaigns",
+          "version": "2025",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
@@ -189,12 +575,46 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-ubi",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent",
-        "nginx-plus-module-appprotect"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-module-plus",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-plugin",
+          "version": "6.16.0",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
@@ -203,12 +623,51 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-ubi8",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent",
-        "nginx-plus-module-appprotect"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-attack-signatures",
+          "version": "2025",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-threat-campaigns",
+          "version": "2025",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
@@ -217,12 +676,46 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-ubi8",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-agent",
-        "nginx-plus-module-appprotect"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-module-plus",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-plugin",
+          "version": "6.16.0",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
@@ -231,11 +724,36 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress:edge-ubi",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-plus-module-appprotectdos"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotectdos",
+          "version": "34+4",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-dos",
+          "version": "34+4",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
@@ -244,13 +762,61 @@
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress:edge-ubi",
       "packages": [
-        "nginx-plus",
-        "nginx-plus-module-njs",
-        "nginx-plus-module-otel",
-        "nginx-plus-module-fips-check",
-        "nginx-plus-module-appprotect",
-        "nginx-plus-module-appprotectdos",
-        "nginx-agent"
+        {
+          "name": "nginx-plus",
+          "version": "34-2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-njs",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-otel",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-fips-check",
+          "version": "34",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-plus-module-appprotectdos",
+          "version": "34+4",
+          "arch": "x86_64"
+        },
+        {
+          "name": "nginx-agent",
+          "version": "2",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect",
+          "version": "34+5.442",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-attack-signatures",
+          "version": "2025",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-threat-campaigns",
+          "version": "2025",
+          "arch": "x86_64"
+        },
+        {
+          "name": "app-protect-dos",
+          "version": "34+4",
+          "arch": "x86_64"
+        }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
diff --git a/tests/suite/test_docker_packages.py b/tests/suite/test_docker_packages.py
index 943714349c..db1105f159 100644
--- a/tests/suite/test_docker_packages.py
+++ b/tests/suite/test_docker_packages.py
@@ -9,10 +9,13 @@
 for image in images["images"]:
     regexInstalled = image["regex"]
     for package in image["packages"]:
-        command = f"{image['cmd']} {package}"
+        command = f"{image['cmd']} {package['name']}"
         output = client.containers.run(
             image["image"], command, entrypoint="", platform=image["platform"], auto_remove=True, detach=False
         )
         result = re.search(regexInstalled, output.decode("utf-8").strip())
-        assert result, f"{package} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
+        assert result, f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
+        assert result.group(2).startswith(
+            package["version"]
+        ), f"{package['name']} version {package['version']} does not match {result.group(2)}"
         print(image["image"], result.group(1, 2, 3))

From f8c26eeebd8afc334598f06651331bb8fd7dc05c Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Fri, 4 Jul 2025 16:59:07 +0100
Subject: [PATCH 03/20] add test to ci

---
 .github/workflows/ci.yml            | 48 +++++++++++++++++++++++++++++
 tests/requirements.txt              |  6 ++++
 tests/suite/test_docker_packages.py |  2 ++
 3 files changed, 56 insertions(+)
 mode change 100644 => 100755 tests/suite/test_docker_packages.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c438c0b1f8..43354a8b98 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -377,6 +377,54 @@ jobs:
     secrets: inherit
     if: ${{ inputs.force || (needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false') || (needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.stable_image_exists != 'true' && needs.checks.outputs.docs_only == 'false') }}
 
+  package-tests:
+    if: ${{ needs.checks.outputs.docs_only != 'true' }}
+    name: Package Tests
+    runs-on: ubuntu-22.04
+    needs: [checks, binaries, build-docker, build-docker-plus, build-docker-nap]
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
+          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Login to GCR
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Install Python dependencies
+        run: |
+          make -f tests/Makefile setup-venv
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Run tests
+        id: packages
+        run: |
+          source tests/venv/bin/activate
+          python tests/suite/test_docker_packages.py | tee package_output.txt 2&>1
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Add comment
+        run: |
+          gh pr comment ${{ github.event.pull_request.number }} -F package_output.txt
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && github.event.pull_request }}
+
   helm-tests:
     if: ${{ needs.checks.outputs.docs_only != 'true' && (inputs.run_tests && inputs.run_tests || true) }}
     name: Helm Tests ${{ matrix.base-os }}
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 9f6bf55043..2137354f42 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -229,6 +229,10 @@ cryptography==45.0.5 \
     # via
     #   -r requirements.in
     #   pyopenssl
+docker==7.1.0 \
+    --hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \
+    --hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0
+    # via -r requirements.in
 durationpy==0.10 \
     --hash=sha256:1fa6893409a6e739c9c72334fc65cca1f355dbdd93405d30f726deb5bde42fba \
     --hash=sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286
@@ -689,6 +693,7 @@ requests==2.32.4 \
     --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422
     # via
     #   -r requirements.in
+    #   docker
     #   forcediphttpsadapter
     #   kubernetes
     #   requests-oauthlib
@@ -722,6 +727,7 @@ urllib3==2.5.0 \
     --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc
     # via
     #   -r requirements.in
+    #   docker
     #   kubernetes
     #   requests
 websocket-client==1.8.0 \
diff --git a/tests/suite/test_docker_packages.py b/tests/suite/test_docker_packages.py
old mode 100644
new mode 100755
index db1105f159..7370abc554
--- a/tests/suite/test_docker_packages.py
+++ b/tests/suite/test_docker_packages.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 import json
 import re
 

From 0ded1d16b9fd4b99373be7d20813f9130d5170fb Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 15:06:26 +0100
Subject: [PATCH 04/20] move package test script

---
 .github/workflows/ci.yml                                        | 2 +-
 .../test_container_packages.py}                                 | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{suite/test_docker_packages.py => scripts/test_container_packages.py} (100%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 43354a8b98..a50aa11f39 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -415,7 +415,7 @@ jobs:
         id: packages
         run: |
           source tests/venv/bin/activate
-          python tests/suite/test_docker_packages.py | tee package_output.txt 2&>1
+          python tests/scripts/test_container_packages.py | tee package_output.txt 2&>1
         if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
 
       - name: Add comment
diff --git a/tests/suite/test_docker_packages.py b/tests/scripts/test_container_packages.py
similarity index 100%
rename from tests/suite/test_docker_packages.py
rename to tests/scripts/test_container_packages.py

From 5fa0f2ed290a2ed701b32fbd640724f05ccbda43 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 15:10:48 +0100
Subject: [PATCH 05/20] fix file open issue

---
 tests/scripts/test_container_packages.py | 31 +++++++++++++-----------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/tests/scripts/test_container_packages.py b/tests/scripts/test_container_packages.py
index 7370abc554..c56d55fd62 100755
--- a/tests/scripts/test_container_packages.py
+++ b/tests/scripts/test_container_packages.py
@@ -1,23 +1,26 @@
 #!/usr/bin/env python
 
 import json
+import os
 import re
 
 import docker
 
 client = docker.from_env()
-images = json.loads(open("tests/data/modules/data.json").read())
+script_dir = os.path.dirname(os.path.abspath(__file__))
+with open(f"{script_dir}../data/modules/data.json") as file:
+    images = json.load(file)
 
-for image in images["images"]:
-    regexInstalled = image["regex"]
-    for package in image["packages"]:
-        command = f"{image['cmd']} {package['name']}"
-        output = client.containers.run(
-            image["image"], command, entrypoint="", platform=image["platform"], auto_remove=True, detach=False
-        )
-        result = re.search(regexInstalled, output.decode("utf-8").strip())
-        assert result, f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
-        assert result.group(2).startswith(
-            package["version"]
-        ), f"{package['name']} version {package['version']} does not match {result.group(2)}"
-        print(image["image"], result.group(1, 2, 3))
+    for image in images["images"]:
+        regexInstalled = image["regex"]
+        for package in image["packages"]:
+            command = f"{image['cmd']} {package['name']}"
+            output = client.containers.run(
+                image["image"], command, entrypoint="", platform=image["platform"], auto_remove=True, detach=False
+            )
+            result = re.search(regexInstalled, output.decode("utf-8").strip())
+            assert result, f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
+            assert result.group(2).startswith(
+                package["version"]
+            ), f"{package['name']} version {package['version']} does not match {result.group(2)}"
+            print(image["image"], result.group(1, 2, 3))

From c77470f0979b2963a8d6cd4b4a593caa59a22626 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 15:14:59 +0100
Subject: [PATCH 06/20] allow setup-venv to be run outside the tests directory

---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 8fe5f96035..80a9ac807d 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -56,7 +56,7 @@ $(MINIKUBE_KUBE_CONFIG_FOLDER): $(KUBE_CONFIG_FOLDER)
 setup-venv:
 	$(PYTHON) -m venv $(ROOT_DIR)/tests/venv;
 	source $(ROOT_DIR)/tests/venv/bin/activate \
-	&& pip install --require-hashes -r requirements.txt --no-deps \
+	&& pip install --require-hashes -r $(ROOT_DIR)/tests/requirements.txt --no-deps \
 	&& playwright install --with-deps chromium
 
 

From 8bbb48e344ab5f339cf6bdaf3ef43901ee71d7fc Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 15:18:52 +0100
Subject: [PATCH 07/20] rename script

---
 .github/workflows/ci.yml                                        | 2 +-
 .pre-commit-config.yaml                                         | 2 +-
 .../{test_container_packages.py => check_container_packages.py} | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename tests/scripts/{test_container_packages.py => check_container_packages.py} (93%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a50aa11f39..44fe654d23 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -415,7 +415,7 @@ jobs:
         id: packages
         run: |
           source tests/venv/bin/activate
-          python tests/scripts/test_container_packages.py | tee package_output.txt 2&>1
+          python tests/scripts/check_container_packages.py | tee package_output.txt 2&>1
         if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
 
       - name: Add comment
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 82d1981cfa..4bf48df7ba 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,7 +27,7 @@ repos:
         args: [--fix=lf]
       - id: name-tests-test
         args: [--pytest-test-first]
-        exclude: ^(tests/suite/utils|tests/suite/fixtures|tests/suite/grpc|tests/settings.py)
+        exclude: ^(tests/suite/utils|tests/suite/fixtures|tests/suite/grpc|tests/settings.py|tests/scripts)
       - id: no-commit-to-branch
       - id: requirements-txt-fixer
       - id: fix-byte-order-marker
diff --git a/tests/scripts/test_container_packages.py b/tests/scripts/check_container_packages.py
similarity index 93%
rename from tests/scripts/test_container_packages.py
rename to tests/scripts/check_container_packages.py
index c56d55fd62..c199e871d0 100755
--- a/tests/scripts/test_container_packages.py
+++ b/tests/scripts/check_container_packages.py
@@ -8,7 +8,7 @@
 
 client = docker.from_env()
 script_dir = os.path.dirname(os.path.abspath(__file__))
-with open(f"{script_dir}../data/modules/data.json") as file:
+with open(f"{script_dir}/../data/modules/data.json") as file:
     images = json.load(file)
 
     for image in images["images"]:

From 026c8938c8eae28b7a4fd1b54683ee585373bbd0 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 16:19:49 +0100
Subject: [PATCH 08/20] allow tag to be passed as cli arg

---
 .github/workflows/ci.yml                  |  2 +-
 tests/data/modules/data.json              | 57 +++++++++++++++--------
 tests/scripts/check_container_packages.py | 40 ++++++++++++++--
 3 files changed, 76 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 44fe654d23..c111cfb2f7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -415,7 +415,7 @@ jobs:
         id: packages
         run: |
           source tests/venv/bin/activate
-          python tests/scripts/check_container_packages.py | tee package_output.txt 2&>1
+          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }} | tee package_output.txt 2&>1
         if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
 
       - name: Add comment
diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
index 3185a13ad8..43939b922d 100644
--- a/tests/data/modules/data.json
+++ b/tests/data/modules/data.json
@@ -1,7 +1,8 @@
 {
   "images": [
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress",
+      "tag_suffix": "",
       "packages": [
         {
           "name": "nginx",
@@ -29,7 +30,8 @@
       "platform": "linux/arm64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
+      "tag_suffix": "",
       "packages": [
         {
           "name": "nginx-plus",
@@ -62,7 +64,8 @@
       "platform": "linux/arm64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
+      "tag_suffix": "",
       "packages": [
         {
           "name": "nginx-plus",
@@ -115,7 +118,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
+      "tag_suffix": "",
       "packages": [
         {
           "name": "nginx-plus",
@@ -163,7 +167,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress:edge",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress",
+      "tag_suffix": "",
       "packages": [
         {
           "name": "nginx-plus",
@@ -201,7 +206,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress:edge",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress",
+      "tag_suffix": "",
       "packages": [
         {
           "name": "nginx-plus",
@@ -264,7 +270,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge-alpine",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress",
+      "tag_suffix": "-alpine",
       "packages": [
         {
           "name": "nginx",
@@ -292,7 +299,8 @@
       "platform": "linux/arm64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-alpine",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
+      "tag_suffix": "-alpine",
       "packages": [
         {
           "name": "nginx-plus",
@@ -325,7 +333,8 @@
       "platform": "linux/arm64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-alpine-fips",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
+      "tag_suffix": "-alpine-fips",
       "packages": [
         {
           "name": "nginx-plus",
@@ -358,7 +367,8 @@
       "platform": "linux/arm64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-alpine-fips",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
+      "tag_suffix": "-alpine-fips",
       "packages": [
         {
           "name": "nginx-plus",
@@ -411,7 +421,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-alpine-fips",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
+      "tag_suffix": "-alpine-fips",
       "packages": [
         {
           "name": "nginx-plus",
@@ -459,7 +470,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress:edge-ubi",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress",
+      "tag_suffix": "-ubi",
       "packages": [
         {
           "name": "nginx",
@@ -487,7 +499,8 @@
       "platform": "linux/arm64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress:edge-ubi",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
+      "tag_suffix": "-ubi",
       "packages": [
         {
           "name": "nginx-plus",
@@ -520,7 +533,8 @@
       "platform": "linux/arm64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-ubi",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
+      "tag_suffix": "-ubi",
       "packages": [
         {
           "name": "nginx-plus",
@@ -573,7 +587,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-ubi",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
+      "tag_suffix": "-ubi",
       "packages": [
         {
           "name": "nginx-plus",
@@ -621,7 +636,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress:edge-ubi8",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
+      "tag_suffix": "-ubi8",
       "packages": [
         {
           "name": "nginx-plus",
@@ -674,7 +690,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress:edge-ubi8",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
+      "tag_suffix": "-ubi8",
       "packages": [
         {
           "name": "nginx-plus",
@@ -722,7 +739,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress:edge-ubi",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress",
+      "tag_suffix": "-ubi",
       "packages": [
         {
           "name": "nginx-plus",
@@ -760,7 +778,8 @@
       "platform": "linux/amd64"
     },
     {
-      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress:edge-ubi",
+      "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress",
+      "tag_suffix": "-ubi",
       "packages": [
         {
           "name": "nginx-plus",
diff --git a/tests/scripts/check_container_packages.py b/tests/scripts/check_container_packages.py
index c199e871d0..9488b84f18 100755
--- a/tests/scripts/check_container_packages.py
+++ b/tests/scripts/check_container_packages.py
@@ -1,10 +1,17 @@
 #!/usr/bin/env python
 
+import argparse
 import json
 import os
 import re
 
 import docker
+import docker.errors
+
+# parse args
+parser = argparse.ArgumentParser()
+parser.add_argument("-t", "--tag", type=str, help="NGINX Ingress Controller image tag", default="edge")
+args = parser.parse_args()
 
 client = docker.from_env()
 script_dir = os.path.dirname(os.path.abspath(__file__))
@@ -13,11 +20,38 @@
 
     for image in images["images"]:
         regexInstalled = image["regex"]
+        tag = f"{args.tag}{image['tag_suffix']}"
+        try:
+            client.images.get(f"{image['image']}:{tag}")
+            print(f"Image {image['image']}:{tag} already exists, skipping pull")
+        except docker.errors.ImageNotFound:
+            print(f"Image {image['image']}:{tag} not found, pulling...")
+            ## pull the image
+            print(f"Pulling image {image['image']}:{tag} for platform {image['platform']}")
+            i = client.images.pull(repository=image["image"], tag=tag, platform=image["platform"])
+            print(f"Image {i.id} pulled successfully")
         for package in image["packages"]:
             command = f"{image['cmd']} {package['name']}"
-            output = client.containers.run(
-                image["image"], command, entrypoint="", platform=image["platform"], auto_remove=True, detach=False
-            )
+            output = ""
+            try:
+                output = client.containers.run(
+                    f"{image['image']}:{tag}",
+                    command,
+                    entrypoint="",
+                    platform=image["platform"],
+                    auto_remove=True,
+                    detach=False,
+                )
+            except (docker.errors.ContainerError, docker.errors.NotFound) as e:
+                print(f"Container error: {e}, retrying")
+                output = client.containers.run(
+                    f"{image['image']}:{tag}",
+                    command,
+                    entrypoint="",
+                    platform=image["platform"],
+                    auto_remove=True,
+                    detach=False,
+                )
             result = re.search(regexInstalled, output.decode("utf-8").strip())
             assert result, f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
             assert result.group(2).startswith(

From 68c6ef10b2fc665acbb62fcc322f2dbb5fb7fa18 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 16:33:34 +0100
Subject: [PATCH 09/20] debug

---
 .github/workflows/ci.yml | 352 +--------------------------------------
 1 file changed, 1 insertion(+), 351 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c111cfb2f7..970756924d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -415,7 +415,7 @@ jobs:
         id: packages
         run: |
           source tests/venv/bin/activate
-          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }} | tee package_output.txt 2&>1
+          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }}
         if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
 
       - name: Add comment
@@ -424,353 +424,3 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && github.event.pull_request }}
-
-  helm-tests:
-    if: ${{ needs.checks.outputs.docs_only != 'true' && (inputs.run_tests && inputs.run_tests || true) }}
-    name: Helm Tests ${{ matrix.base-os }}
-    runs-on: ubuntu-22.04
-    needs: [checks, binaries, build-docker, build-docker-plus]
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - base-os: debian
-            image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress
-            tag: ${{ needs.checks.outputs.build_tag }}
-            type: oss
-          - base-os: debian-plus
-            image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress
-            tag: ${{ needs.checks.outputs.build_tag }}
-            type: plus
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
-        with:
-          token_format: access_token
-          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
-          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Login to GCR
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
-        with:
-          registry: gcr.io
-          username: oauth2accesstoken
-          password: ${{ steps.auth.outputs.access_token }}
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Check if stable image exists
-        id: stable_exists
-        run: |
-          if docker pull ${{ matrix.image }}:${{ needs.checks.outputs.stable_tag }}; then
-            echo "exists=true" >> $GITHUB_OUTPUT
-          fi
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Pull build image
-        run: |
-          docker pull ${{ matrix.image }}:${{ needs.checks.outputs.build_tag }}
-        if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && steps.stable_exists.outputs.exists != 'true' }}
-
-      - name: Fetch Cached Artifacts
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
-        with:
-          path: ${{ github.workspace }}/dist
-          key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }}
-        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
-        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Build Docker Image ${{ matrix.base-os }}
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
-        with:
-          file: build/Dockerfile
-          context: "."
-          cache-from: type=gha,scope=${{ matrix.base-os }}
-          target: goreleaser
-          tags: "${{ matrix.image }}:${{ matrix.tag }}"
-          pull: true
-          load: true
-          build-args: |
-            BUILD_OS=${{ matrix.base-os }}
-            IC_VERSION=CI
-          secrets: |
-            ${{ matrix.type == 'plus' && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
-            ${{ matrix.type == 'plus' && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }}
-        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Deploy Kubernetes
-        id: k8s
-        run: |
-          kind create cluster --name ${{ github.run_id }} --image=kindest/node:v${{ needs.checks.outputs.k8s_latest }} --wait 75s
-          docker version
-          kind load docker-image "${{ matrix.image }}:${{ matrix.tag }}" --name ${{ github.run_id }}
-          echo "DEBUG: Kind setup complete!"
-        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Create Plus Secret
-        run: kubectl create secret generic license-token --from-literal=license.jwt="${{ secrets.PLUS_JWT }}" --type="nginx.com/license"
-        if: ${{ matrix.type == 'plus' && steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Install Chart
-        run: >
-          helm install
-          ${{ matrix.type }}
-          .
-          --set controller.image.repository=${{ matrix.image }}
-          --set controller.image.tag=${{ matrix.tag }}
-          --set controller.service.type=NodePort
-          --set controller.nginxplus=${{ contains(matrix.type, 'plus') && 'true' || 'false' }}
-          --set controller.telemetryReporting.enable=false
-          --wait
-        working-directory: ${{ github.workspace }}/charts/nginx-ingress
-        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Expose Test Ingresses
-        run: |
-          kubectl port-forward service/${{ matrix.type }}-nginx-ingress-controller 8080:80 8443:443 &
-        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Test HTTP
-        run: |
-          counter=0
-          max_attempts=5
-          until [ $(curl --write-out %{http_code} -s --output /dev/null http://localhost:8080) -eq 404 ]; do
-            if [ ${counter} -eq ${max_attempts} ]; then
-              exit 1
-            fi
-            printf '.'; counter=$(($counter+1)); sleep 5;
-          done
-        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Test HTTPS
-        run: |
-          counter=0
-          max_attempts=5
-          until [ $(curl --write-out %{http_code} -ks --output /dev/null https://localhost:8443) -eq 000 ]; do
-            if [ ${counter} -eq ${max_attempts} ]; then
-              exit 1
-            fi
-            printf '.'; counter=$(($counter+1)); sleep 5;
-          done
-        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
-
-  setup-matrix:
-    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
-    name: Setup Matrix for Smoke Tests
-    runs-on: ubuntu-22.04
-    needs: [binaries, checks]
-    permissions:
-      contents: read
-      id-token: write
-    outputs:
-      matrix_oss: ${{ steps.set-matrix.outputs.matrix_oss }}
-      matrix_plus: ${{ steps.set-matrix.outputs.matrix_plus }}
-      matrix_nap: ${{ steps.set-matrix.outputs.matrix_nap }}
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - id: set-matrix
-        run: |
-          echo "matrix_oss=$(cat .github/data/matrix-smoke-oss.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
-          echo "matrix_plus=$(cat .github/data/matrix-smoke-plus.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
-          echo "matrix_nap=$(cat .github/data/matrix-smoke-nap.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
-
-      - name: Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
-
-      - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
-        with:
-          token_format: access_token
-          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
-          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Login to GCR
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
-        with:
-          registry: gcr.io
-          username: oauth2accesstoken
-          password: ${{ steps.auth.outputs.access_token }}
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Check if test image exists
-        id: check-image
-        run: |
-          docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}
-        shell: bash
-        continue-on-error: true
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
-
-      - name: Build Test-Runner Container
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
-        with:
-          file: tests/Dockerfile
-          context: "."
-          cache-from: type=gha,scope=test-runner
-          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
-          pull: true
-          push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
-          load: false
-        if: ${{ steps.check-image.outcome == 'failure' && needs.checks.outputs.docs_only == 'false' }}
-
-  smoke-tests-oss:
-    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
-    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
-    needs:
-      - checks
-      - setup-matrix
-      - build-docker
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_oss) }}
-    permissions:
-      contents: read
-      id-token: write
-    uses: ./.github/workflows/setup-smoke.yml
-    secrets: inherit
-    with:
-      image: ${{ matrix.images.image }}
-      target: ${{ matrix.images.target }}
-      nap-modules: ${{ matrix.images.nap_modules }}
-      marker: ${{ matrix.images.marker }}
-      label: ${{ matrix.images.label }}
-      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
-      build-tag: ${{ needs.checks.outputs.build_tag }}
-      stable-tag: ${{ needs.checks.outputs.stable_tag }}
-      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
-      k8s-version: ${{ matrix.k8s }}
-
-  smoke-tests-plus:
-    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
-    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
-    needs:
-      - checks
-      - setup-matrix
-      - build-docker-plus
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_plus) }}
-    permissions:
-      contents: read
-      id-token: write
-    uses: ./.github/workflows/setup-smoke.yml
-    secrets: inherit
-    with:
-      image: ${{ matrix.images.image }}
-      target: ${{ matrix.images.target }}
-      nap-modules: ${{ matrix.images.nap_modules }}
-      marker: ${{ matrix.images.marker }}
-      label: ${{ matrix.images.label }}
-      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
-      build-tag: ${{ needs.checks.outputs.build_tag }}
-      stable-tag: ${{ needs.checks.outputs.stable_tag }}
-      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
-      k8s-version: ${{ matrix.k8s }}
-
-  smoke-tests-nap:
-    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
-    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
-    needs:
-      - checks
-      - setup-matrix
-      - build-docker-nap
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_nap) }}
-    permissions:
-      contents: read
-      id-token: write
-    uses: ./.github/workflows/setup-smoke.yml
-    secrets: inherit
-    with:
-      image: ${{ matrix.images.image }}
-      target: ${{ matrix.images.target }}
-      nap-modules: ${{ matrix.images.nap_modules }}
-      marker: ${{ matrix.images.marker }}
-      label: ${{ matrix.images.label }}
-      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
-      build-tag: ${{ needs.checks.outputs.build_tag }}
-      stable-tag: ${{ needs.checks.outputs.stable_tag }}
-      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
-      k8s-version: ${{ matrix.k8s }}
-
-  tag-stable:
-    name: Tag tested image as stable
-    needs: [checks, build-docker, build-docker-plus, build-docker-nap, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap]
-    permissions:
-      contents: read # To checkout repository
-      id-token: write # To sign into Google Container Registry
-    uses: ./.github/workflows/retag-images.yml
-    with:
-      source_tag: ${{ needs.checks.outputs.build_tag }}
-      target_tag: ${{ needs.checks.outputs.stable_tag }}
-      dry_run: false
-    secrets: inherit
-    if: ${{ inputs.force || (needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false') || (needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.stable_image_exists != 'true' && needs.checks.outputs.docs_only == 'false') }}
-
-  final-results:
-    if: ${{ !cancelled() }}
-    runs-on: ubuntu-22.04
-    name: Final CI Results
-    needs: [tag-stable, build-docker, build-docker-plus, build-docker-nap, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap]
-    steps:
-      - run: |
-          tagResult="${{ needs.tag-stable.result }}"
-          smokeOSSResult="${{ needs.smoke-tests-oss.result }}"
-          smokePlusResult="${{ needs.smoke-tests-plus.result }}"
-          smokeNAPResult="${{ needs.smoke-tests-nap.result }}"
-          buildOSSResult="${{ needs.build-docker.result }}"
-          buildPlusResult="${{ needs.build-docker-plus.result }}"
-          buildNAPResult="${{ needs.build-docker-nap.result }}"
-          if [[ $tagResult != "success" && $tagResult != "skipped" ]]; then
-            exit 1
-          fi
-          if [[ $smokeOSSResult != "success" && $smokeOSSResult != "skipped" ]]; then
-            exit 1
-          fi
-          if [[ $smokePlusResult != "success" && $smokePlusResult != "skipped" ]]; then
-            exit 1
-          fi
-          if [[ $smokeNAPResult != "success" && $smokeNAPResult != "skipped" ]]; then
-            exit 1
-          fi
-          if [[ $buildOSSResult != "success" && $buildOSSResult != "skipped" ]]; then
-            exit 1
-          fi
-          if [[ $buildPlusResult != "success" && $buildPlusResult != "skipped" ]]; then
-            exit 1
-          fi
-          if [[ $buildNAPResult != "success" && $buildNAPResult != "skipped" ]]; then
-            exit 1
-          fi
-
-  trigger-image-promotion:
-    name: Promote images on Force Run
-    needs:
-      - build-docker
-      - build-docker-plus
-      - build-docker-nap
-      - final-results
-    permissions:
-      contents: write # for pushing to Helm Charts repository
-      id-token: write # To sign into Google Container Registry
-      actions: read
-      packages: write # for helm to push to GHCR
-      security-events: write
-      pull-requests: write # for scout report
-    uses: ./.github/workflows/image-promotion.yml
-    secrets: inherit
-    if: ${{ inputs.force && inputs.force || false }}

From e6596a73e8252fefaa59a893ab4f8831c3c2f0c7 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 16:39:35 +0100
Subject: [PATCH 10/20] add qemu

---
 .github/workflows/ci.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 970756924d..a3131b47c0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -389,6 +389,12 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+        with:
+          platforms: arm64
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
       - name: Authenticate to Google Cloud
         id: auth
         uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10

From dd6f7987317caf8357df7ad3cd33b31805055d95 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 16:42:16 +0100
Subject: [PATCH 11/20] switch to amd64

---
 .github/workflows/ci.yml     |  6 ------
 tests/data/modules/data.json | 10 +++++-----
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a3131b47c0..970756924d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -389,12 +389,6 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
-        with:
-          platforms: arm64
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
-
       - name: Authenticate to Google Cloud
         id: auth
         uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
index 43939b922d..2c1675cbb1 100644
--- a/tests/data/modules/data.json
+++ b/tests/data/modules/data.json
@@ -7,27 +7,27 @@
         {
           "name": "nginx",
           "version": "1.27.5",
-          "arch": "arm64"
+          "arch": "amd64"
         },
         {
           "name": "nginx-module-njs",
           "version": "1.27.5",
-          "arch": "arm64"
+          "arch": "amd64"
         },
         {
           "name": "nginx-module-otel",
           "version": "1.27.5",
-          "arch": "arm64"
+          "arch": "amd64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "arm64"
+          "arch": "amd64"
         }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
-      "platform": "linux/arm64"
+      "platform": "linux/amd64"
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",

From b371040a84cb541a0beb8c90aee718aeb4fc1bad Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 16:48:01 +0100
Subject: [PATCH 12/20] switch to amd64

---
 tests/data/modules/data.json | 68 ++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
index 2c1675cbb1..9f80df8b16 100644
--- a/tests/data/modules/data.json
+++ b/tests/data/modules/data.json
@@ -36,32 +36,32 @@
         {
           "name": "nginx-plus",
           "version": "34-2",
-          "arch": "arm64"
+          "arch": "amd64"
         },
         {
           "name": "nginx-plus-module-njs",
           "version": "34",
-          "arch": "arm64"
+          "arch": "amd64"
         },
         {
           "name": "nginx-plus-module-otel",
           "version": "34",
-          "arch": "arm64"
+          "arch": "amd64"
         },
         {
           "name": "nginx-plus-module-fips-check",
           "version": "34",
-          "arch": "arm64"
+          "arch": "amd64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "arm64"
+          "arch": "amd64"
         }
       ],
       "cmd": "dpkg -l",
       "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
-      "platform": "linux/arm64"
+      "platform": "linux/amd64"
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
@@ -276,27 +276,27 @@
         {
           "name": "nginx",
           "version": "1.27.5",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-module-njs",
           "version": "1.27.5",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-module-otel",
           "version": "1.27.5",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "aarch64"
+          "arch": "x64_64"
         }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
-      "platform": "linux/arm64"
+      "platform": "linux/amd64"
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
@@ -305,32 +305,32 @@
         {
           "name": "nginx-plus",
           "version": "34-r2",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-njs",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-otel",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-fips-check",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "aarch64"
+          "arch": "x64_64"
         }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
-      "platform": "linux/arm64"
+      "platform": "linux/amd64"
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
@@ -339,32 +339,32 @@
         {
           "name": "nginx-plus",
           "version": "34-r2",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-njs",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-otel",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-fips-check",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "aarch64"
+          "arch": "x64_64"
         }
       ],
       "cmd": "apk list",
       "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
-      "platform": "linux/arm64"
+      "platform": "linux/amd64"
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
@@ -476,27 +476,27 @@
         {
           "name": "nginx",
           "version": "1.27.5",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-module-njs",
           "version": "1.27.5",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-module-otel",
           "version": "1.27.5",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "aarch64"
+          "arch": "x64_64"
         }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
-      "platform": "linux/arm64"
+      "platform": "linux/amd64"
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
@@ -505,32 +505,32 @@
         {
           "name": "nginx-plus",
           "version": "34-2",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-njs",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-otel",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-plus-module-fips-check",
           "version": "34",
-          "arch": "aarch64"
+          "arch": "x64_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "aarch64"
+          "arch": "x64_64"
         }
       ],
       "cmd": "rpm -q",
       "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
-      "platform": "linux/arm64"
+      "platform": "linux/amd64"
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",

From c2c5dc303ba6bed3df9aae52ec9d1b399f6da043 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 16:53:21 +0100
Subject: [PATCH 13/20] add tests back

---
 .github/workflows/ci.yml | 352 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 351 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 970756924d..c111cfb2f7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -415,7 +415,7 @@ jobs:
         id: packages
         run: |
           source tests/venv/bin/activate
-          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }}
+          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }} | tee package_output.txt 2&>1
         if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
 
       - name: Add comment
@@ -424,3 +424,353 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && github.event.pull_request }}
+
+  helm-tests:
+    if: ${{ needs.checks.outputs.docs_only != 'true' && (inputs.run_tests && inputs.run_tests || true) }}
+    name: Helm Tests ${{ matrix.base-os }}
+    runs-on: ubuntu-22.04
+    needs: [checks, binaries, build-docker, build-docker-plus]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - base-os: debian
+            image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress
+            tag: ${{ needs.checks.outputs.build_tag }}
+            type: oss
+          - base-os: debian-plus
+            image: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress
+            tag: ${{ needs.checks.outputs.build_tag }}
+            type: plus
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
+          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Login to GCR
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Check if stable image exists
+        id: stable_exists
+        run: |
+          if docker pull ${{ matrix.image }}:${{ needs.checks.outputs.stable_tag }}; then
+            echo "exists=true" >> $GITHUB_OUTPUT
+          fi
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Pull build image
+        run: |
+          docker pull ${{ matrix.image }}:${{ needs.checks.outputs.build_tag }}
+        if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && steps.stable_exists.outputs.exists != 'true' }}
+
+      - name: Fetch Cached Artifacts
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ${{ github.workspace }}/dist
+          key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Build Docker Image ${{ matrix.base-os }}
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          file: build/Dockerfile
+          context: "."
+          cache-from: type=gha,scope=${{ matrix.base-os }}
+          target: goreleaser
+          tags: "${{ matrix.image }}:${{ matrix.tag }}"
+          pull: true
+          load: true
+          build-args: |
+            BUILD_OS=${{ matrix.base-os }}
+            IC_VERSION=CI
+          secrets: |
+            ${{ matrix.type == 'plus' && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
+            ${{ matrix.type == 'plus' && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          kind create cluster --name ${{ github.run_id }} --image=kindest/node:v${{ needs.checks.outputs.k8s_latest }} --wait 75s
+          docker version
+          kind load docker-image "${{ matrix.image }}:${{ matrix.tag }}" --name ${{ github.run_id }}
+          echo "DEBUG: Kind setup complete!"
+        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Create Plus Secret
+        run: kubectl create secret generic license-token --from-literal=license.jwt="${{ secrets.PLUS_JWT }}" --type="nginx.com/license"
+        if: ${{ matrix.type == 'plus' && steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Install Chart
+        run: >
+          helm install
+          ${{ matrix.type }}
+          .
+          --set controller.image.repository=${{ matrix.image }}
+          --set controller.image.tag=${{ matrix.tag }}
+          --set controller.service.type=NodePort
+          --set controller.nginxplus=${{ contains(matrix.type, 'plus') && 'true' || 'false' }}
+          --set controller.telemetryReporting.enable=false
+          --wait
+        working-directory: ${{ github.workspace }}/charts/nginx-ingress
+        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Expose Test Ingresses
+        run: |
+          kubectl port-forward service/${{ matrix.type }}-nginx-ingress-controller 8080:80 8443:443 &
+        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Test HTTP
+        run: |
+          counter=0
+          max_attempts=5
+          until [ $(curl --write-out %{http_code} -s --output /dev/null http://localhost:8080) -eq 404 ]; do
+            if [ ${counter} -eq ${max_attempts} ]; then
+              exit 1
+            fi
+            printf '.'; counter=$(($counter+1)); sleep 5;
+          done
+        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Test HTTPS
+        run: |
+          counter=0
+          max_attempts=5
+          until [ $(curl --write-out %{http_code} -ks --output /dev/null https://localhost:8443) -eq 000 ]; do
+            if [ ${counter} -eq ${max_attempts} ]; then
+              exit 1
+            fi
+            printf '.'; counter=$(($counter+1)); sleep 5;
+          done
+        if: ${{ steps.stable_exists.outputs.exists != 'true' && needs.checks.outputs.docs_only == 'false' }}
+
+  setup-matrix:
+    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
+    name: Setup Matrix for Smoke Tests
+    runs-on: ubuntu-22.04
+    needs: [binaries, checks]
+    permissions:
+      contents: read
+      id-token: write
+    outputs:
+      matrix_oss: ${{ steps.set-matrix.outputs.matrix_oss }}
+      matrix_plus: ${{ steps.set-matrix.outputs.matrix_plus }}
+      matrix_nap: ${{ steps.set-matrix.outputs.matrix_nap }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - id: set-matrix
+        run: |
+          echo "matrix_oss=$(cat .github/data/matrix-smoke-oss.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
+          echo "matrix_plus=$(cat .github/data/matrix-smoke-plus.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
+          echo "matrix_nap=$(cat .github/data/matrix-smoke-nap.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
+          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Login to GCR
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Check if test image exists
+        id: check-image
+        run: |
+          docker pull gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}
+        shell: bash
+        continue-on-error: true
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.docs_only == 'false' }}
+
+      - name: Build Test-Runner Container
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
+        with:
+          file: tests/Dockerfile
+          context: "."
+          cache-from: type=gha,scope=test-runner
+          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
+          pull: true
+          push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
+          load: false
+        if: ${{ steps.check-image.outcome == 'failure' && needs.checks.outputs.docs_only == 'false' }}
+
+  smoke-tests-oss:
+    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
+    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
+    needs:
+      - checks
+      - setup-matrix
+      - build-docker
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_oss) }}
+    permissions:
+      contents: read
+      id-token: write
+    uses: ./.github/workflows/setup-smoke.yml
+    secrets: inherit
+    with:
+      image: ${{ matrix.images.image }}
+      target: ${{ matrix.images.target }}
+      nap-modules: ${{ matrix.images.nap_modules }}
+      marker: ${{ matrix.images.marker }}
+      label: ${{ matrix.images.label }}
+      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
+      build-tag: ${{ needs.checks.outputs.build_tag }}
+      stable-tag: ${{ needs.checks.outputs.stable_tag }}
+      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
+      k8s-version: ${{ matrix.k8s }}
+
+  smoke-tests-plus:
+    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
+    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
+    needs:
+      - checks
+      - setup-matrix
+      - build-docker-plus
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_plus) }}
+    permissions:
+      contents: read
+      id-token: write
+    uses: ./.github/workflows/setup-smoke.yml
+    secrets: inherit
+    with:
+      image: ${{ matrix.images.image }}
+      target: ${{ matrix.images.target }}
+      nap-modules: ${{ matrix.images.nap_modules }}
+      marker: ${{ matrix.images.marker }}
+      label: ${{ matrix.images.label }}
+      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
+      build-tag: ${{ needs.checks.outputs.build_tag }}
+      stable-tag: ${{ needs.checks.outputs.stable_tag }}
+      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
+      k8s-version: ${{ matrix.k8s }}
+
+  smoke-tests-nap:
+    if: ${{ inputs.force || (inputs.run_tests && inputs.run_tests || true) || needs.checks.outputs.docs_only != 'true' }}
+    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
+    needs:
+      - checks
+      - setup-matrix
+      - build-docker-nap
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.setup-matrix.outputs.matrix_nap) }}
+    permissions:
+      contents: read
+      id-token: write
+    uses: ./.github/workflows/setup-smoke.yml
+    secrets: inherit
+    with:
+      image: ${{ matrix.images.image }}
+      target: ${{ matrix.images.target }}
+      nap-modules: ${{ matrix.images.nap_modules }}
+      marker: ${{ matrix.images.marker }}
+      label: ${{ matrix.images.label }}
+      go-md5: ${{ needs.checks.outputs.go_code_md5 }}
+      build-tag: ${{ needs.checks.outputs.build_tag }}
+      stable-tag: ${{ needs.checks.outputs.stable_tag }}
+      authenticated: ${{ needs.checks.outputs.forked_workflow != 'true' }}
+      k8s-version: ${{ matrix.k8s }}
+
+  tag-stable:
+    name: Tag tested image as stable
+    needs: [checks, build-docker, build-docker-plus, build-docker-nap, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap]
+    permissions:
+      contents: read # To checkout repository
+      id-token: write # To sign into Google Container Registry
+    uses: ./.github/workflows/retag-images.yml
+    with:
+      source_tag: ${{ needs.checks.outputs.build_tag }}
+      target_tag: ${{ needs.checks.outputs.stable_tag }}
+      dry_run: false
+    secrets: inherit
+    if: ${{ inputs.force || (needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false') || (needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.stable_image_exists != 'true' && needs.checks.outputs.docs_only == 'false') }}
+
+  final-results:
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-22.04
+    name: Final CI Results
+    needs: [tag-stable, build-docker, build-docker-plus, build-docker-nap, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap]
+    steps:
+      - run: |
+          tagResult="${{ needs.tag-stable.result }}"
+          smokeOSSResult="${{ needs.smoke-tests-oss.result }}"
+          smokePlusResult="${{ needs.smoke-tests-plus.result }}"
+          smokeNAPResult="${{ needs.smoke-tests-nap.result }}"
+          buildOSSResult="${{ needs.build-docker.result }}"
+          buildPlusResult="${{ needs.build-docker-plus.result }}"
+          buildNAPResult="${{ needs.build-docker-nap.result }}"
+          if [[ $tagResult != "success" && $tagResult != "skipped" ]]; then
+            exit 1
+          fi
+          if [[ $smokeOSSResult != "success" && $smokeOSSResult != "skipped" ]]; then
+            exit 1
+          fi
+          if [[ $smokePlusResult != "success" && $smokePlusResult != "skipped" ]]; then
+            exit 1
+          fi
+          if [[ $smokeNAPResult != "success" && $smokeNAPResult != "skipped" ]]; then
+            exit 1
+          fi
+          if [[ $buildOSSResult != "success" && $buildOSSResult != "skipped" ]]; then
+            exit 1
+          fi
+          if [[ $buildPlusResult != "success" && $buildPlusResult != "skipped" ]]; then
+            exit 1
+          fi
+          if [[ $buildNAPResult != "success" && $buildNAPResult != "skipped" ]]; then
+            exit 1
+          fi
+
+  trigger-image-promotion:
+    name: Promote images on Force Run
+    needs:
+      - build-docker
+      - build-docker-plus
+      - build-docker-nap
+      - final-results
+    permissions:
+      contents: write # for pushing to Helm Charts repository
+      id-token: write # To sign into Google Container Registry
+      actions: read
+      packages: write # for helm to push to GHCR
+      security-events: write
+      pull-requests: write # for scout report
+    uses: ./.github/workflows/image-promotion.yml
+    secrets: inherit
+    if: ${{ inputs.force && inputs.force || false }}

From 53de46e8cac238c283ab99ca5d1b7bfbec11e3b9 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 17:03:14 +0100
Subject: [PATCH 14/20] add pull_request: write permission to add comment

---
 .github/workflows/ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c111cfb2f7..d6bd09885b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -384,6 +384,7 @@ jobs:
     needs: [checks, binaries, build-docker, build-docker-plus, build-docker-nap]
     permissions:
       contents: read
+      pull-requests: write # for package report
       id-token: write
     steps:
       - name: Checkout Repository
@@ -415,7 +416,7 @@ jobs:
         id: packages
         run: |
           source tests/venv/bin/activate
-          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }} | tee package_output.txt 2&>1
+          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }} | tee package_output.txt
         if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
 
       - name: Add comment

From 25ca39698b5d19eb8f5d08d4a062df0b4fe5ff64 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Tue, 15 Jul 2025 17:37:34 +0100
Subject: [PATCH 15/20] add logger

---
 .github/workflows/ci.yml                  |  2 +-
 tests/data/modules/data.json              | 46 ++++++++++-----------
 tests/scripts/check_container_packages.py | 49 ++++++++++++++++++-----
 3 files changed, 62 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d6bd09885b..73fe990913 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -416,7 +416,7 @@ jobs:
         id: packages
         run: |
           source tests/venv/bin/activate
-          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }} | tee package_output.txt
+          python tests/scripts/check_container_packages.py --tag ${{ needs.checks.outputs.build_tag }} --log package_output.txt
         if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
 
       - name: Add comment
diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
index 9f80df8b16..de64923c2f 100644
--- a/tests/data/modules/data.json
+++ b/tests/data/modules/data.json
@@ -276,22 +276,22 @@
         {
           "name": "nginx",
           "version": "1.27.5",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-module-njs",
           "version": "1.27.5",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-module-otel",
           "version": "1.27.5",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "x64_64"
+          "arch": "x86_64"
         }
       ],
       "cmd": "apk list",
@@ -305,27 +305,27 @@
         {
           "name": "nginx-plus",
           "version": "34-r2",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-njs",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-otel",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-fips-check",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "x64_64"
+          "arch": "x86_64"
         }
       ],
       "cmd": "apk list",
@@ -339,27 +339,27 @@
         {
           "name": "nginx-plus",
           "version": "34-r2",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-njs",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-otel",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-fips-check",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "x64_64"
+          "arch": "x86_64"
         }
       ],
       "cmd": "apk list",
@@ -476,22 +476,22 @@
         {
           "name": "nginx",
           "version": "1.27.5",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-module-njs",
           "version": "1.27.5",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-module-otel",
           "version": "1.27.5",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "x64_64"
+          "arch": "x86_64"
         }
       ],
       "cmd": "rpm -q",
@@ -505,27 +505,27 @@
         {
           "name": "nginx-plus",
           "version": "34-2",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-njs",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-otel",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-plus-module-fips-check",
           "version": "34",
-          "arch": "x64_64"
+          "arch": "x86_64"
         },
         {
           "name": "nginx-agent",
           "version": "3.0",
-          "arch": "x64_64"
+          "arch": "x86_64"
         }
       ],
       "cmd": "rpm -q",
diff --git a/tests/scripts/check_container_packages.py b/tests/scripts/check_container_packages.py
index 9488b84f18..2498bf8e3d 100755
--- a/tests/scripts/check_container_packages.py
+++ b/tests/scripts/check_container_packages.py
@@ -2,6 +2,7 @@
 
 import argparse
 import json
+import logging
 import os
 import re
 
@@ -11,8 +12,26 @@
 # parse args
 parser = argparse.ArgumentParser()
 parser.add_argument("-t", "--tag", type=str, help="NGINX Ingress Controller image tag", default="edge")
+parser.add_argument("-l", "--log", type=str, help="log file", required=False)
 args = parser.parse_args()
 
+# Create a logger
+logger = logging.getLogger("package_checker")
+logger.setLevel(logging.DEBUG)
+
+# Create a stream handler (for stdout)
+stream_handler = logging.StreamHandler()
+stream_handler.setLevel(logging.DEBUG)
+stream_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+logger.addHandler(stream_handler)
+
+if args.log:
+    # Create a file handler
+    file_handler = logging.FileHandler(args.log)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(logging.Formatter("%(message)s"))
+    logger.addHandler(file_handler)
+
 client = docker.from_env()
 script_dir = os.path.dirname(os.path.abspath(__file__))
 with open(f"{script_dir}/../data/modules/data.json") as file:
@@ -22,14 +41,17 @@
         regexInstalled = image["regex"]
         tag = f"{args.tag}{image['tag_suffix']}"
         try:
-            client.images.get(f"{image['image']}:{tag}")
-            print(f"Image {image['image']}:{tag} already exists, skipping pull")
+            i = client.images.get(f"{image['image']}:{tag}")
+            ## check if the image is for the correct platform
+            if image["platform"] != i.attrs["Os"]:
+                raise docker.errors.ImageNotFound(
+                    f"Image {image['image']}:{tag} is not for platform {image['platform']}, found {i.attrs['Os']}"
+                )
         except docker.errors.ImageNotFound:
-            print(f"Image {image['image']}:{tag} not found, pulling...")
             ## pull the image
-            print(f"Pulling image {image['image']}:{tag} for platform {image['platform']}")
+            logger.debug(f"Pulling image {image['image']}:{tag} for platform {image['platform']}")
             i = client.images.pull(repository=image["image"], tag=tag, platform=image["platform"])
-            print(f"Image {i.id} pulled successfully")
+            logger.debug(f"Image {i.id} pulled successfully")
         for package in image["packages"]:
             command = f"{image['cmd']} {package['name']}"
             output = ""
@@ -43,7 +65,7 @@
                     detach=False,
                 )
             except (docker.errors.ContainerError, docker.errors.NotFound) as e:
-                print(f"Container error: {e}, retrying")
+                logger.error(f"{e}, retrying")
                 output = client.containers.run(
                     f"{image['image']}:{tag}",
                     command,
@@ -53,8 +75,13 @@
                     detach=False,
                 )
             result = re.search(regexInstalled, output.decode("utf-8").strip())
-            assert result, f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
-            assert result.group(2).startswith(
-                package["version"]
-            ), f"{package['name']} version {package['version']} does not match {result.group(2)}"
-            print(image["image"], result.group(1, 2, 3))
+            assert result, logger.error(
+                f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
+            )
+            assert result.group(2).startswith(package["version"]), logger.error(
+                f"{package['name']} version {package['version']} does not match {result.group(2)}"
+            )
+            assert result.group(3) == package["arch"], logger.error(
+                f"{package['name']} arch {package['arch']} does not match {result.group(3)}"
+            )
+            logger.info(f"{image["image"]}, {result.group(1)}, {result.group(2)}, {result.group(3)}")

From 4219a46a287b8d24fa43d72a877e8af506336b4a Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Wed, 16 Jul 2025 10:20:12 +0100
Subject: [PATCH 16/20] move regex and cmd to python code

---
 tests/data/modules/data.json              | 459 ++++++++--------------
 tests/scripts/check_container_packages.py |  25 +-
 2 files changed, 172 insertions(+), 312 deletions(-)

diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
index de64923c2f..4873fb948d 100644
--- a/tests/data/modules/data.json
+++ b/tests/data/modules/data.json
@@ -6,27 +6,22 @@
       "packages": [
         {
           "name": "nginx",
-          "version": "1.27.5",
-          "arch": "amd64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-module-njs",
-          "version": "1.27.5",
-          "arch": "amd64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-module-otel",
-          "version": "1.27.5",
-          "arch": "amd64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-agent",
-          "version": "3.0",
-          "arch": "amd64"
+          "version": "3.0"
         }
       ],
-      "cmd": "dpkg -l",
-      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "system": "debian",
       "platform": "linux/amd64"
     },
     {
@@ -35,32 +30,26 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "amd64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "3.0",
-          "arch": "amd64"
+          "version": "3.0"
         }
       ],
-      "cmd": "dpkg -l",
-      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "system": "debian",
       "platform": "linux/amd64"
     },
     {
@@ -69,52 +58,42 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "amd64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "amd64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect",
-          "version": "34+5.442",
-          "arch": "amd64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-attack-signatures",
-          "version": "2025",
-          "arch": "amd64"
+          "version": "2025"
         },
         {
           "name": "app-protect-threat-campaigns",
-          "version": "2025",
-          "arch": "amd64"
+          "version": "2025"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "amd64"
+          "version": "2"
         }
       ],
-      "cmd": "dpkg -l",
-      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "system": "debian",
       "platform": "linux/amd64"
     },
     {
@@ -123,47 +102,38 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "amd64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "amd64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-module-plus",
-          "version": "34+5.442",
-          "arch": "amd64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-plugin",
-          "version": "6.16.0",
-          "arch": "amd64"
+          "version": "6.16.0"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "amd64"
+          "version": "2"
         }
       ],
-      "cmd": "dpkg -l",
-      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "system": "debian",
       "platform": "linux/amd64"
     },
     {
@@ -172,37 +142,30 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "amd64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-appprotectdos",
-          "version": "34+4",
-          "arch": "amd64"
+          "version": "34+4"
         },
         {
           "name": "app-protect-dos",
-          "version": "34+4",
-          "arch": "amd64"
+          "version": "34+4"
         }
       ],
-      "cmd": "dpkg -l",
-      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "system": "debian",
       "platform": "linux/amd64"
     },
     {
@@ -211,62 +174,50 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "amd64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "amd64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "amd64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect",
-          "version": "34+5.442",
-          "arch": "amd64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-attack-signatures",
-          "version": "2025",
-          "arch": "amd64"
+          "version": "2025"
         },
         {
           "name": "app-protect-threat-campaigns",
-          "version": "2025",
-          "arch": "amd64"
+          "version": "2025"
         },
         {
           "name": "nginx-plus-module-appprotectdos",
-          "version": "34+4",
-          "arch": "amd64"
+          "version": "34+4"
         },
         {
           "name": "app-protect-dos",
-          "version": "34+4",
-          "arch": "amd64"
+          "version": "34+4"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "amd64"
+          "version": "2"
         }
       ],
-      "cmd": "dpkg -l",
-      "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+      "system": "debian",
       "platform": "linux/amd64"
     },
     {
@@ -275,27 +226,22 @@
       "packages": [
         {
           "name": "nginx",
-          "version": "1.27.5",
-          "arch": "x86_64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-module-njs",
-          "version": "1.27.5",
-          "arch": "x86_64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-module-otel",
-          "version": "1.27.5",
-          "arch": "x86_64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-agent",
-          "version": "3.0",
-          "arch": "x86_64"
+          "version": "3.0"
         }
       ],
-      "cmd": "apk list",
-      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "system": "alpine",
       "platform": "linux/amd64"
     },
     {
@@ -304,32 +250,26 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-r2",
-          "arch": "x86_64"
+          "version": "34-r2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "3.0",
-          "arch": "x86_64"
+          "version": "3.0"
         }
       ],
-      "cmd": "apk list",
-      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "system": "alpine",
       "platform": "linux/amd64"
     },
     {
@@ -338,32 +278,26 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-r2",
-          "arch": "x86_64"
+          "version": "34-r2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "3.0",
-          "arch": "x86_64"
+          "version": "3.0"
         }
       ],
-      "cmd": "apk list",
-      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "system": "alpine",
       "platform": "linux/amd64"
     },
     {
@@ -372,52 +306,42 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-r2",
-          "arch": "x86_64"
+          "version": "34-r2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "x86_64"
+          "version": "2"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34.5.442",
-          "arch": "x86_64"
+          "version": "34.5.442"
         },
         {
           "name": "app-protect",
-          "version": "34.5.442",
-          "arch": "x86_64"
+          "version": "34.5.442"
         },
         {
           "name": "app-protect-attack-signatures",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         },
         {
           "name": "app-protect-threat-campaigns",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         }
       ],
-      "cmd": "apk list",
-      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "system": "alpine",
       "platform": "linux/amd64"
     },
     {
@@ -426,47 +350,38 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-r2",
-          "arch": "x86_64"
+          "version": "34-r2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "x86_64"
+          "version": "2"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34.5.442",
-          "arch": "x86_64"
+          "version": "34.5.442"
         },
         {
           "name": "app-protect-module-plus",
-          "version": "34.5.442",
-          "arch": "x86_64"
+          "version": "34.5.442"
         },
         {
           "name": "app-protect-plugin",
-          "version": "6.16.0",
-          "arch": "x86_64"
+          "version": "6.16.0"
         }
       ],
-      "cmd": "apk list",
-      "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+      "system": "alpine",
       "platform": "linux/amd64"
     },
     {
@@ -475,27 +390,22 @@
       "packages": [
         {
           "name": "nginx",
-          "version": "1.27.5",
-          "arch": "x86_64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-module-njs",
-          "version": "1.27.5",
-          "arch": "x86_64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-module-otel",
-          "version": "1.27.5",
-          "arch": "x86_64"
+          "version": "1.27.5"
         },
         {
           "name": "nginx-agent",
-          "version": "3.0",
-          "arch": "x86_64"
+          "version": "3.0"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     },
     {
@@ -504,32 +414,26 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "x86_64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "3.0",
-          "arch": "x86_64"
+          "version": "3.0"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     },
     {
@@ -538,52 +442,42 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "x86_64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "x86_64"
+          "version": "2"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-attack-signatures",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         },
         {
           "name": "app-protect-threat-campaigns",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     },
     {
@@ -592,47 +486,38 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "x86_64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "x86_64"
+          "version": "2"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-module-plus",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-plugin",
-          "version": "6.16.0",
-          "arch": "x86_64"
+          "version": "6.16.0"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     },
     {
@@ -641,52 +526,42 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "x86_64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "x86_64"
+          "version": "2"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-attack-signatures",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         },
         {
           "name": "app-protect-threat-campaigns",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     },
     {
@@ -695,47 +570,38 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "x86_64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "x86_64"
+          "version": "2"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-module-plus",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-plugin",
-          "version": "6.16.0",
-          "arch": "x86_64"
+          "version": "6.16.0"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     },
     {
@@ -744,37 +610,30 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "x86_64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-appprotectdos",
-          "version": "34+4",
-          "arch": "x86_64"
+          "version": "34+4"
         },
         {
           "name": "app-protect-dos",
-          "version": "34+4",
-          "arch": "x86_64"
+          "version": "34+4"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     },
     {
@@ -783,62 +642,50 @@
       "packages": [
         {
           "name": "nginx-plus",
-          "version": "34-2",
-          "arch": "x86_64"
+          "version": "34-2"
         },
         {
           "name": "nginx-plus-module-njs",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-otel",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-fips-check",
-          "version": "34",
-          "arch": "x86_64"
+          "version": "34"
         },
         {
           "name": "nginx-plus-module-appprotect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "nginx-plus-module-appprotectdos",
-          "version": "34+4",
-          "arch": "x86_64"
+          "version": "34+4"
         },
         {
           "name": "nginx-agent",
-          "version": "2",
-          "arch": "x86_64"
+          "version": "2"
         },
         {
           "name": "app-protect",
-          "version": "34+5.442",
-          "arch": "x86_64"
+          "version": "34+5.442"
         },
         {
           "name": "app-protect-attack-signatures",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         },
         {
           "name": "app-protect-threat-campaigns",
-          "version": "2025",
-          "arch": "x86_64"
+          "version": "2025"
         },
         {
           "name": "app-protect-dos",
-          "version": "34+4",
-          "arch": "x86_64"
+          "version": "34+4"
         }
       ],
-      "cmd": "rpm -q",
-      "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+      "system": "ubi",
       "platform": "linux/amd64"
     }
   ]
diff --git a/tests/scripts/check_container_packages.py b/tests/scripts/check_container_packages.py
index 2498bf8e3d..909d5d2107 100755
--- a/tests/scripts/check_container_packages.py
+++ b/tests/scripts/check_container_packages.py
@@ -32,13 +32,29 @@
     file_handler.setFormatter(logging.Formatter("%(message)s"))
     logger.addHandler(file_handler)
 
+systems = {
+    "alpine": {
+        "cmd": "apk list",
+        "regex": "^(.+?)-(\\d+.+?)\\s+(\\w+).*\\[installed\\]",
+    },
+    "debian": {
+        "cmd": "dpkg -l",
+        "regex": "ii\\s+(.+?)\\s+(.+?)\\s+(\\w+?)\\s",
+    },
+    "ubi": {
+        "cmd": "rpm -q",
+        "regex": "(.+?)-(\\d+.+)(?:\\.ngx)?\\.(\\w+)",
+    },
+}
+
 client = docker.from_env()
 script_dir = os.path.dirname(os.path.abspath(__file__))
 with open(f"{script_dir}/../data/modules/data.json") as file:
     images = json.load(file)
 
     for image in images["images"]:
-        regexInstalled = image["regex"]
+        regexInstalled = systems[image["system"]]["regex"]
+        cmd = systems[image["system"]]["cmd"]
         tag = f"{args.tag}{image['tag_suffix']}"
         try:
             i = client.images.get(f"{image['image']}:{tag}")
@@ -53,7 +69,7 @@
             i = client.images.pull(repository=image["image"], tag=tag, platform=image["platform"])
             logger.debug(f"Image {i.id} pulled successfully")
         for package in image["packages"]:
-            command = f"{image['cmd']} {package['name']}"
+            command = f"{cmd} {package['name']}"
             output = ""
             try:
                 output = client.containers.run(
@@ -81,7 +97,4 @@
             assert result.group(2).startswith(package["version"]), logger.error(
                 f"{package['name']} version {package['version']} does not match {result.group(2)}"
             )
-            assert result.group(3) == package["arch"], logger.error(
-                f"{package['name']} arch {package['arch']} does not match {result.group(3)}"
-            )
-            logger.info(f"{image["image"]}, {result.group(1)}, {result.group(2)}, {result.group(3)}")
+            logger.info(f"{image["image"]}, {result.group(1)}, {result.group(2)}")

From 86f6610f038445fc01d30592375c682523cbd42b Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Wed, 16 Jul 2025 11:21:18 +0100
Subject: [PATCH 17/20] check each image architecture

---
 .github/workflows/ci.yml                  |  6 ++
 tests/data/modules/data.json              | 83 +++++++++++++++++-----
 tests/scripts/check_container_packages.py | 85 ++++++++++++-----------
 3 files changed, 113 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 73fe990913..cbf325b1da 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -390,6 +390,12 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+        with:
+          platforms: arm64
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' }}
+
       - name: Authenticate to Google Cloud
         id: auth
         uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10
diff --git a/tests/data/modules/data.json b/tests/data/modules/data.json
index 4873fb948d..b30eb79c72 100644
--- a/tests/data/modules/data.json
+++ b/tests/data/modules/data.json
@@ -22,7 +22,10 @@
         }
       ],
       "system": "debian",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
@@ -50,7 +53,10 @@
         }
       ],
       "system": "debian",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
@@ -94,7 +100,9 @@
         }
       ],
       "system": "debian",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
@@ -134,7 +142,9 @@
         }
       ],
       "system": "debian",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress",
@@ -166,7 +176,9 @@
         }
       ],
       "system": "debian",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress",
@@ -218,7 +230,9 @@
         }
       ],
       "system": "debian",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress",
@@ -242,7 +256,10 @@
         }
       ],
       "system": "alpine",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
@@ -270,7 +287,10 @@
         }
       ],
       "system": "alpine",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
@@ -298,7 +318,10 @@
         }
       ],
       "system": "alpine",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
@@ -342,7 +365,9 @@
         }
       ],
       "system": "alpine",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
@@ -382,7 +407,9 @@
         }
       ],
       "system": "alpine",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-ingress",
@@ -406,7 +433,10 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic/nginx-plus-ingress",
@@ -434,7 +464,10 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
@@ -478,7 +511,9 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
@@ -518,7 +553,9 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap/nginx-plus-ingress",
@@ -562,7 +599,9 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-nap-v5/nginx-plus-ingress",
@@ -602,7 +641,9 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos/nginx-plus-ingress",
@@ -634,7 +675,9 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     },
     {
       "image": "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-dos-nap/nginx-plus-ingress",
@@ -686,7 +729,9 @@
         }
       ],
       "system": "ubi",
-      "platform": "linux/amd64"
+      "platforms": [
+        "linux/amd64"
+      ]
     }
   ]
 }
diff --git a/tests/scripts/check_container_packages.py b/tests/scripts/check_container_packages.py
index 909d5d2107..4e69326ae0 100755
--- a/tests/scripts/check_container_packages.py
+++ b/tests/scripts/check_container_packages.py
@@ -53,48 +53,49 @@
     images = json.load(file)
 
     for image in images["images"]:
-        regexInstalled = systems[image["system"]]["regex"]
-        cmd = systems[image["system"]]["cmd"]
-        tag = f"{args.tag}{image['tag_suffix']}"
-        try:
-            i = client.images.get(f"{image['image']}:{tag}")
-            ## check if the image is for the correct platform
-            if image["platform"] != i.attrs["Os"]:
-                raise docker.errors.ImageNotFound(
-                    f"Image {image['image']}:{tag} is not for platform {image['platform']}, found {i.attrs['Os']}"
-                )
-        except docker.errors.ImageNotFound:
-            ## pull the image
-            logger.debug(f"Pulling image {image['image']}:{tag} for platform {image['platform']}")
-            i = client.images.pull(repository=image["image"], tag=tag, platform=image["platform"])
-            logger.debug(f"Image {i.id} pulled successfully")
-        for package in image["packages"]:
-            command = f"{cmd} {package['name']}"
-            output = ""
+        for platform in image["platforms"]:
+            regexInstalled = systems[image["system"]]["regex"]
+            cmd = systems[image["system"]]["cmd"]
+            tag = f"{args.tag}{image['tag_suffix']}"
             try:
-                output = client.containers.run(
-                    f"{image['image']}:{tag}",
-                    command,
-                    entrypoint="",
-                    platform=image["platform"],
-                    auto_remove=True,
-                    detach=False,
+                i = client.images.get(f"{image['image']}:{tag}")
+                ## check if the image is for the correct platform
+                if platform != i.attrs["Os"]:
+                    raise docker.errors.ImageNotFound(
+                        f"Image {image['image']}:{tag} is not for platform {platform}, found {i.attrs['Os']}"
+                    )
+            except docker.errors.ImageNotFound:
+                ## pull the image
+                logger.debug(f"Pulling image {image['image']}:{tag} for platform {platform}")
+                i = client.images.pull(repository=image["image"], tag=tag, platform=platform)
+                logger.debug(f"Image {i.id} pulled successfully")
+            for package in image["packages"]:
+                command = f"{cmd} {package['name']}"
+                output = ""
+                try:
+                    output = client.containers.run(
+                        f"{image['image']}:{tag}",
+                        command,
+                        entrypoint="",
+                        platform=platform,
+                        auto_remove=True,
+                        detach=False,
+                    )
+                except (docker.errors.ContainerError, docker.errors.NotFound) as e:
+                    logger.error(f"{e}, retrying")
+                    output = client.containers.run(
+                        f"{image['image']}:{tag}",
+                        command,
+                        entrypoint="",
+                        platform=platform,
+                        auto_remove=True,
+                        detach=False,
+                    )
+                result = re.search(regexInstalled, output.decode("utf-8").strip())
+                assert result, logger.error(
+                    f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
                 )
-            except (docker.errors.ContainerError, docker.errors.NotFound) as e:
-                logger.error(f"{e}, retrying")
-                output = client.containers.run(
-                    f"{image['image']}:{tag}",
-                    command,
-                    entrypoint="",
-                    platform=image["platform"],
-                    auto_remove=True,
-                    detach=False,
+                assert result.group(2).startswith(package["version"]), logger.error(
+                    f"{package['name']} version {package['version']} does not match {result.group(2)}"
                 )
-            result = re.search(regexInstalled, output.decode("utf-8").strip())
-            assert result, logger.error(
-                f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
-            )
-            assert result.group(2).startswith(package["version"]), logger.error(
-                f"{package['name']} version {package['version']} does not match {result.group(2)}"
-            )
-            logger.info(f"{image["image"]}, {result.group(1)}, {result.group(2)}")
+                logger.info(f"{image["image"]}, {result.group(1)}, {result.group(2)}, {result.group(3)}")

From c269e0217b0e40b60148306fa59d4c570fbcca93 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Wed, 16 Jul 2025 11:35:48 +0100
Subject: [PATCH 18/20] use named variables for image name

---
 tests/scripts/check_container_packages.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/scripts/check_container_packages.py b/tests/scripts/check_container_packages.py
index 4e69326ae0..abcb0f50f8 100755
--- a/tests/scripts/check_container_packages.py
+++ b/tests/scripts/check_container_packages.py
@@ -54,7 +54,8 @@
 
     for image in images["images"]:
         for platform in image["platforms"]:
-            regexInstalled = systems[image["system"]]["regex"]
+            image_name = image["image"]
+            regex = systems[image["system"]]["regex"]
             cmd = systems[image["system"]]["cmd"]
             tag = f"{args.tag}{image['tag_suffix']}"
             try:
@@ -67,7 +68,7 @@
             except docker.errors.ImageNotFound:
                 ## pull the image
                 logger.debug(f"Pulling image {image['image']}:{tag} for platform {platform}")
-                i = client.images.pull(repository=image["image"], tag=tag, platform=platform)
+                i = client.images.pull(repository=image_name, tag=tag, platform=platform)
                 logger.debug(f"Image {i.id} pulled successfully")
             for package in image["packages"]:
                 command = f"{cmd} {package['name']}"
@@ -91,11 +92,11 @@
                         auto_remove=True,
                         detach=False,
                     )
-                result = re.search(regexInstalled, output.decode("utf-8").strip())
+                result = re.search(regex, output.decode("utf-8").strip())
                 assert result, logger.error(
                     f"{package['name']} not found in {image['image']}, output: {output.decode('utf-8').strip()}"
                 )
                 assert result.group(2).startswith(package["version"]), logger.error(
                     f"{package['name']} version {package['version']} does not match {result.group(2)}"
                 )
-                logger.info(f"{image["image"]}, {result.group(1)}, {result.group(2)}, {result.group(3)}")
+                logger.info(f"{image_name}, {result.group(1)}, {result.group(2)}, {result.group(3)}")

From 4fa7908d8ad71b1baae06f0ed5b91eb3954d2fa8 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Wed, 16 Jul 2025 11:52:13 +0100
Subject: [PATCH 19/20] format comment

---
 .github/workflows/ci.yml                  | 7 ++++++-
 tests/scripts/check_container_packages.py | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cbf325b1da..2f37557e20 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -427,7 +427,12 @@ jobs:
 
       - name: Add comment
         run: |
-          gh pr comment ${{ github.event.pull_request.number }} -F package_output.txt
+          # make sure the comment is formatted correctly, as a code block
+          echo '### Package Report' > output.txt
+          echo '```' >> output.txt
+          cat package_output.txt >> output.txt
+          echo '```' >> output.txt
+          gh pr comment ${{ github.event.pull_request.number }} -F output.txt
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && github.event.pull_request }}
diff --git a/tests/scripts/check_container_packages.py b/tests/scripts/check_container_packages.py
index abcb0f50f8..f4010da4fc 100755
--- a/tests/scripts/check_container_packages.py
+++ b/tests/scripts/check_container_packages.py
@@ -99,4 +99,4 @@
                 assert result.group(2).startswith(package["version"]), logger.error(
                     f"{package['name']} version {package['version']} does not match {result.group(2)}"
                 )
-                logger.info(f"{image_name}, {result.group(1)}, {result.group(2)}, {result.group(3)}")
+                logger.info(f"{image_name}:{tag}, {result.group(1)}, {result.group(2)}, {result.group(3)}")

From 0713c16d9ae40458bb61b6667f28d7e5c8a16d92 Mon Sep 17 00:00:00 2001
From: Paul Abel <p.abel@f5.com>
Date: Wed, 16 Jul 2025 11:58:21 +0100
Subject: [PATCH 20/20] update comment if it already exists

---
 .github/workflows/ci.yml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2f37557e20..6265b3155a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -378,7 +378,7 @@ jobs:
     if: ${{ inputs.force || (needs.checks.outputs.forked_workflow == 'true' && needs.checks.outputs.docs_only == 'false') || (needs.checks.outputs.forked_workflow == 'false' && needs.checks.outputs.stable_image_exists != 'true' && needs.checks.outputs.docs_only == 'false') }}
 
   package-tests:
-    if: ${{ needs.checks.outputs.docs_only != 'true' }}
+    if: ${{ needs.checks.outputs.docs_only != 'true' && (inputs.run_tests && inputs.run_tests || true) }}
     name: Package Tests
     runs-on: ubuntu-22.04
     needs: [checks, binaries, build-docker, build-docker-plus, build-docker-nap]
@@ -432,7 +432,7 @@ jobs:
           echo '```' >> output.txt
           cat package_output.txt >> output.txt
           echo '```' >> output.txt
-          gh pr comment ${{ github.event.pull_request.number }} -F output.txt
+          gh pr comment --edit-last --create-if-none ${{ github.event.pull_request.number }} -F output.txt
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         if: ${{ ( needs.checks.outputs.forked_workflow == 'false' || needs.checks.outputs.docs_only == 'false' ) && github.event.pull_request }}
@@ -737,10 +737,12 @@ jobs:
     if: ${{ !cancelled() }}
     runs-on: ubuntu-22.04
     name: Final CI Results
-    needs: [tag-stable, build-docker, build-docker-plus, build-docker-nap, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap]
+    needs: [tag-stable, build-docker, build-docker-plus, build-docker-nap, smoke-tests-oss, smoke-tests-plus, smoke-tests-nap, package-tests, helm-tests]
     steps:
       - run: |
           tagResult="${{ needs.tag-stable.result }}"
+          packageResult="${{ needs.package-tests.result }}"
+          helmResult="${{ needs.helm-tests.result }}"
           smokeOSSResult="${{ needs.smoke-tests-oss.result }}"
           smokePlusResult="${{ needs.smoke-tests-plus.result }}"
           smokeNAPResult="${{ needs.smoke-tests-nap.result }}"
@@ -768,6 +770,12 @@ jobs:
           if [[ $buildNAPResult != "success" && $buildNAPResult != "skipped" ]]; then
             exit 1
           fi
+          if [[ $helmResult != "success" && $helmResult != "skipped" ]]; then
+            exit 1
+          fi
+          if [[ $packageResult != "success" && $packageResult != "skipped" ]]; then
+            exit 1
+          fi
 
   trigger-image-promotion:
     name: Promote images on Force Run