From f89b56169e7bcb9e4a2972b6d9e26f80f9a0bf71 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Wed, 22 Oct 2025 00:38:09 +0200 Subject: [PATCH 01/18] IT Tests to cover thread pool monitoring. Some failing. --- .../main/test/app/monitoring/SlowServlet.java | 40 ++ .../main/test/app/monitoring/TestServlet.java | 34 ++ .../monitoring/ThreadPoolMonitoringTest.java | 516 ++++++++++++++++++ 3 files changed, 590 insertions(+) create mode 100644 appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java create mode 100644 appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java create mode 100644 appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java new file mode 100644 index 00000000000..02bf5b2206d --- /dev/null +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +package org.glassfish.main.test.app.monitoring; + +import java.io.IOException; +import java.io.PrintWriter; + +import jakarta.servlet.ServletException; +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +@WebServlet("/slow") +public class SlowServlet extends HttpServlet { + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + try { + Thread.sleep(2000); // 2 second delay to keep threads busy + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + resp.setContentType("text/plain"); + try (PrintWriter writer = resp.getWriter()) { + writer.println("Slow response completed"); + writer.println("Thread: " + Thread.currentThread().getName()); + } + } +} diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java new file mode 100644 index 00000000000..cbd133dce6b --- /dev/null +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +package org.glassfish.main.test.app.monitoring; + +import java.io.IOException; +import java.io.PrintWriter; + +import jakarta.servlet.ServletException; +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +@WebServlet("/test") +public class TestServlet extends HttpServlet { + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setContentType("text/plain"); + try (PrintWriter writer = resp.getWriter()) { + writer.println("Thread Pool Test Servlet"); + writer.println("Thread: " + Thread.currentThread().getName()); + } + } +} diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java new file mode 100644 index 00000000000..142f6c37b1d --- /dev/null +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java @@ -0,0 +1,516 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ + +package org.glassfish.main.test.app.monitoring; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.glassfish.main.itest.tools.GlassFishTestEnvironment; +import org.glassfish.main.itest.tools.asadmin.Asadmin; +import org.glassfish.main.itest.tools.asadmin.AsadminResult; +import org.glassfish.main.itest.tools.asadmin.AsadminResultMatcher; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.exporter.ZipExporter; +import org.jboss.shrinkwrap.api.spec.WebArchive; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ThreadPoolMonitoringTest { + + private static final String APP_NAME = "threadpool-test"; + private static final Asadmin ASADMIN = GlassFishTestEnvironment.getAsadmin(); + + @BeforeAll + static void deployApp() throws IOException { + File warFile = createDeployment(); + try { + AsadminResult result = ASADMIN.exec("deploy", warFile.getAbsolutePath()); + assertThat(result, AsadminResultMatcher.asadminOK()); + + // Enable monitoring + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH"); + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=HIGH"); + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.monitoring-enabled=true"); + } finally { + warFile.delete(); + } + } + + @AfterAll + static void undeployApp() { + ASADMIN.exec("undeploy", APP_NAME); + // Disable monitoring + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=OFF"); + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=OFF"); + } + + @Test + void testThreadPoolMetricsUnderLoad() throws Exception { + // Get baseline metrics + ThreadPoolMetrics baseline = getThreadPoolMetrics(); + assertTrue(baseline.currentThreadCount >= 0); + assertTrue(baseline.currentThreadsBusy >= 0); + assertTrue(baseline.currentThreadsBusy <= baseline.currentThreadCount); + + // Generate concurrent load + ExecutorService executor = Executors.newFixedThreadPool(10); + CompletableFuture[] futures = new CompletableFuture[20]; + + for (int i = 0; i < futures.length; i++) { + futures[i] = CompletableFuture.runAsync(() -> { + try { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/slow"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, executor); + } + + // Check metrics during load + Thread.sleep(100); // Let requests start + ThreadPoolMetrics duringLoad = getThreadPoolMetrics(); + + assertThat("Current threads should be >= baseline", + duringLoad.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); + assertThat("Busy threads should be > 0 during load", + duringLoad.currentThreadsBusy, greaterThanOrEqualTo(1)); + assertThat("Busy threads <= current threads", + duringLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadCount)); + assertThat("Current threads <= max threads", + duringLoad.currentThreadCount, lessThanOrEqualTo(duringLoad.maxThreads)); + + // Wait for completion + CompletableFuture.allOf(futures).get(30, TimeUnit.SECONDS); + executor.shutdown(); + + // Check metrics after load + Thread.sleep(1000); // Let threads settle + ThreadPoolMetrics afterLoad = getThreadPoolMetrics(); + + assertThat("Busy threads should decrease after load", + afterLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadsBusy)); + assertTrue(afterLoad.currentThreadsBusy >= 0); + assertTrue(afterLoad.currentThreadsBusy <= afterLoad.currentThreadCount); + } + + @Test + void testThreadPoolMetricsBaseline() throws Exception { + ThreadPoolMetrics metrics = getThreadPoolMetrics(); + + // Basic sanity checks + assertTrue(metrics.currentThreadCount >= 0, "Current thread count should be non-negative"); + assertTrue(metrics.currentThreadsBusy >= 0, "Busy thread count should be non-negative"); + assertTrue(metrics.maxThreads > 0, "Max threads should be positive"); + + // Logical consistency + assertTrue(metrics.currentThreadsBusy <= metrics.currentThreadCount, + "Busy threads should not exceed current threads"); + assertTrue(metrics.currentThreadCount <= metrics.maxThreads, + "Current threads should not exceed max threads"); + } + + @Test + void testThreadPoolMetricsWithSequentialRequests() throws Exception { + ThreadPoolMetrics baseline = getThreadPoolMetrics(); + + // Make sequential requests to see if metrics respond + for (int i = 0; i < 5; i++) { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/test"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + conn.disconnect(); + Thread.sleep(100); + } + + ThreadPoolMetrics afterSequential = getThreadPoolMetrics(); + + // Metrics should remain consistent + assertTrue(afterSequential.currentThreadCount >= 0); + assertTrue(afterSequential.currentThreadsBusy >= 0); + assertTrue(afterSequential.currentThreadsBusy <= afterSequential.currentThreadCount); + } + + @Test + void testThreadPoolMetricsWithBurstLoad() throws Exception { + ThreadPoolMetrics baseline = getThreadPoolMetrics(); + + // Create burst of quick requests + ExecutorService executor = Executors.newFixedThreadPool(50); + CompletableFuture[] futures = new CompletableFuture[100]; + + for (int i = 0; i < futures.length; i++) { + futures[i] = CompletableFuture.runAsync(() -> { + try { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/test"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, executor); + } + + // Check metrics immediately during burst + ThreadPoolMetrics duringBurst = getThreadPoolMetrics(); + + // Wait for completion + CompletableFuture.allOf(futures).get(15, TimeUnit.SECONDS); + executor.shutdown(); + + ThreadPoolMetrics afterBurst = getThreadPoolMetrics(); + + // Validate all metrics remain within bounds + assertTrue(duringBurst.currentThreadCount >= baseline.currentThreadCount); + assertTrue(duringBurst.currentThreadsBusy >= 0); + assertTrue(duringBurst.currentThreadsBusy <= duringBurst.currentThreadCount); + + assertTrue(afterBurst.currentThreadCount >= 0); + assertTrue(afterBurst.currentThreadsBusy >= 0); + assertTrue(afterBurst.currentThreadsBusy <= afterBurst.currentThreadCount); + } + + @Test + void testThreadPoolMetricsConsistency() throws Exception { + // Take multiple samples to check for consistency + ThreadPoolMetrics[] samples = new ThreadPoolMetrics[5]; + + for (int i = 0; i < samples.length; i++) { + samples[i] = getThreadPoolMetrics(); + Thread.sleep(200); + } + + // All samples should have consistent logical relationships + for (ThreadPoolMetrics sample : samples) { + assertTrue(sample.currentThreadCount >= 0, "Current threads >= 0"); + assertTrue(sample.currentThreadsBusy >= 0, "Busy threads >= 0"); + assertTrue(sample.maxThreads > 0, "Max threads > 0"); + assertTrue(sample.currentThreadsBusy <= sample.currentThreadCount, + "Busy <= Current"); + assertTrue(sample.currentThreadCount <= sample.maxThreads, + "Current <= Max"); + } + } + + @Test + void testThreadPoolMetricsUnderSustainedLoad() throws Exception { + ThreadPoolMetrics baseline = getThreadPoolMetrics(); + + // Create sustained load for longer period + ExecutorService executor = Executors.newFixedThreadPool(8); + CompletableFuture[] futures = new CompletableFuture[16]; + + for (int i = 0; i < futures.length; i++) { + futures[i] = CompletableFuture.runAsync(() -> { + try { + // Multiple slow requests per thread + for (int j = 0; j < 3; j++) { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/slow"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + conn.disconnect(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }, executor); + } + + // Sample metrics during sustained load + Thread.sleep(500); + ThreadPoolMetrics sample1 = getThreadPoolMetrics(); + + Thread.sleep(1000); + ThreadPoolMetrics sample2 = getThreadPoolMetrics(); + + Thread.sleep(1000); + ThreadPoolMetrics sample3 = getThreadPoolMetrics(); + + // Wait for completion + CompletableFuture.allOf(futures).get(60, TimeUnit.SECONDS); + executor.shutdown(); + + ThreadPoolMetrics afterSustained = getThreadPoolMetrics(); + + // During sustained load, we should see consistent thread usage + ThreadPoolMetrics[] samples = {sample1, sample2, sample3}; + for (int i = 0; i < samples.length; i++) { + ThreadPoolMetrics sample = samples[i]; + assertTrue(sample.currentThreadCount >= baseline.currentThreadCount, + "Sample " + i + ": threads should increase under load"); + assertTrue(sample.currentThreadsBusy >= 0, + "Sample " + i + ": busy threads should be non-negative"); + assertTrue(sample.currentThreadsBusy <= sample.currentThreadCount, + "Sample " + i + ": busy <= current"); + } + + // After load, busy threads should decrease + assertTrue(afterSustained.currentThreadsBusy <= sample3.currentThreadsBusy, + "Busy threads should decrease after sustained load"); + } + + private static File createDeployment() throws IOException { + WebArchive war = ShrinkWrap.create(WebArchive.class, APP_NAME + ".war") + .addClass(TestServlet.class) + .addClass(SlowServlet.class); + + File warFile = new File(System.getProperty("java.io.tmpdir"), APP_NAME + ".war"); + war.as(ZipExporter.class).exportTo(warFile, true); + return warFile; + } + + private ThreadPoolMetrics getThreadPoolMetrics() { + // Try different possible monitoring paths + String[] possiblePaths = { + "server.network.http-listener-1.thread-pool", + "server.http-service.http-listener.http-listener-1.thread-pool", + "server.applications.application.threadpool-test.thread-pool", + "server.thread-pools.thread-pool.http-thread-pool" + }; + + // First, let's see what monitoring data is actually available + AsadminResult listResult = ASADMIN.exec("list", "*thread*"); + System.out.println("Available thread monitoring paths: " + listResult.getStdOut()); + + // Try to get metrics from the most likely path + AsadminResult currentResult = ASADMIN.exec("get", "-m", + "server.network.http-listener-1.thread-pool.currentthreadcount"); + AsadminResult busyResult = ASADMIN.exec("get", "-m", + "server.network.http-listener-1.thread-pool.currentthreadsbusy"); + AsadminResult maxResult = ASADMIN.exec("get", "-m", + "server.network.http-listener-1.thread-pool.maxthreads"); + + return new ThreadPoolMetrics( + extractValue(currentResult.getStdOut(), "currentthreadcount"), + extractValue(busyResult.getStdOut(), "currentthreadsbusy"), + extractValue(maxResult.getStdOut(), "maxthreads") + ); + } + + private int extractValue(String output, String metric) { + String[] lines = output.split("\n"); + for (String line : lines) { + if (line.contains(metric + "-count")) { + return Integer.parseInt(line.split("=")[1].trim()); + } + } + return 0; + } + + private static class ThreadPoolMetrics { + final int currentThreadCount; + final int currentThreadsBusy; + final int maxThreads; + + ThreadPoolMetrics(int currentThreadCount, int currentThreadsBusy, int maxThreads) { + this.currentThreadCount = currentThreadCount; + this.currentThreadsBusy = currentThreadsBusy; + this.maxThreads = maxThreads; + } + } + + @Test + void testThreadPoolSizeIncrease() throws Exception { + ThreadPoolMetrics initial = getThreadPoolMetrics(); + int originalMaxThreads = initial.maxThreads; + + try { + // Increase pool size significantly + int newMaxThreads = originalMaxThreads + 100; + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + newMaxThreads); + + Thread.sleep(2000); + + ThreadPoolMetrics afterIncrease = getThreadPoolMetrics(); + assertEquals(newMaxThreads, afterIncrease.maxThreads, "Max threads should reflect new configuration"); + + // Generate load to test the increased pool + ExecutorService executor = Executors.newFixedThreadPool(50); + for (int i = 0; i < 50; i++) { + executor.submit(() -> { + try { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/slow?delay=2000"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + } catch (Exception e) { + // Ignore for this test + } + }); + } + + Thread.sleep(500); // Let load start + ThreadPoolMetrics underLoad = getThreadPoolMetrics(); + + // Critical assertions: monitoring should report valid values + assertTrue(underLoad.currentThreadCount >= 0, "Current thread count should never be negative"); + assertTrue(underLoad.currentThreadCount > initial.currentThreadCount, "Thread count should increase under load"); + assertTrue(underLoad.currentThreadCount <= newMaxThreads, "Current <= max"); + + // Should have approximately 50 threads active for 50 concurrent requests + assertTrue(underLoad.currentThreadCount >= 30, "Should have many threads active for 50 concurrent requests"); + + executor.shutdown(); + Thread.sleep(3000); // Wait for completion + + } finally { + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); + } + } + + @Test + void testThreadPoolSizeDecrease() throws Exception { + ThreadPoolMetrics initial = getThreadPoolMetrics(); + int originalMaxThreads = initial.maxThreads; + + try { + // First increase pool size to 100 + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=100"); + Thread.sleep(2000); + + ThreadPoolMetrics afterIncrease = getThreadPoolMetrics(); + assertEquals(100, afterIncrease.maxThreads, "Max threads should be 100"); + + // Generate significant load to utilize the large pool + ExecutorService executor = Executors.newFixedThreadPool(80); + for (int i = 0; i < 80; i++) { + executor.submit(() -> { + try { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/slow?delay=3000"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + } catch (Exception e) { + // Ignore for this test + } + }); + } + + Thread.sleep(1000); // Let load build up + ThreadPoolMetrics underHeavyLoad = getThreadPoolMetrics(); + + // Verify the pool scaled up to handle heavy load + assertTrue(underHeavyLoad.currentThreadCount > initial.currentThreadCount, "Thread count should increase under heavy load"); + assertTrue(underHeavyLoad.currentThreadCount <= 100, "Current threads should not exceed 100"); + assertTrue(underHeavyLoad.currentThreadCount >= 0, "Current thread count should never be negative"); + + // Should have approximately 80 threads active (or close to it) + assertTrue(underHeavyLoad.currentThreadCount >= 50, "Should have many threads active for 80 concurrent requests"); + + // Now decrease pool size to 10 while under load + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=10"); + Thread.sleep(1000); + + ThreadPoolMetrics afterDecrease = getThreadPoolMetrics(); + assertEquals(10, afterDecrease.maxThreads, "Max threads should be 10"); + + // Critical assertions: thread count should remain valid + assertTrue(afterDecrease.currentThreadCount >= 0, "Current thread count should never be negative"); + assertTrue(afterDecrease.currentThreadsBusy >= 0, "Busy threads should be non-negative"); + + // Since we had 80 concurrent requests, current threads should still be high + // (threads don't disappear instantly when pool size is reduced) + assertTrue(afterDecrease.currentThreadCount >= 10, "Should still have many active threads from previous load"); + + executor.shutdown(); + Thread.sleep(4000); // Wait for requests to complete + + ThreadPoolMetrics afterCompletion = getThreadPoolMetrics(); + assertTrue(afterCompletion.currentThreadCount >= 0, "Thread count should never be negative"); + assertTrue(afterCompletion.currentThreadCount <= 10, "Eventually current threads should respect new max"); + + } finally { + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); + } + } + + @Test + void testThreadPoolSizeCycling() throws Exception { + ThreadPoolMetrics initial = getThreadPoolMetrics(); + int originalMaxThreads = initial.maxThreads; + + try { + int[] testSizes = {originalMaxThreads + 3, originalMaxThreads - 1, originalMaxThreads + 7, originalMaxThreads}; + + for (int testSize : testSizes) { + testSize = Math.max(1, testSize); + + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + testSize); + Thread.sleep(1500); + + ThreadPoolMetrics metrics = getThreadPoolMetrics(); + assertEquals(testSize, metrics.maxThreads, "Max threads should match configured size: " + testSize); + assertTrue(metrics.currentThreadCount <= testSize, "Current threads should not exceed max: " + testSize); + assertTrue(metrics.currentThreadsBusy <= metrics.currentThreadCount, "Busy <= current for size: " + testSize); + } + + } finally { + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); + } + } + + @Test + void testThreadPoolSizeUnderLoad() throws Exception { + ThreadPoolMetrics initial = getThreadPoolMetrics(); + int originalMaxThreads = initial.maxThreads; + + try { + // Start with increased pool size + int largePoolSize = originalMaxThreads + 5; + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + largePoolSize); + Thread.sleep(2000); + + // Generate load + ExecutorService executor = Executors.newFixedThreadPool(8); + for (int i = 0; i < 8; i++) { + executor.submit(() -> { + try { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/slow?delay=3000"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + } catch (Exception e) { + // Ignore for this test + } + }); + } + + Thread.sleep(500); + ThreadPoolMetrics duringLoad = getThreadPoolMetrics(); + + // Decrease pool size while under load + int smallPoolSize = Math.max(3, originalMaxThreads - 1); + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + smallPoolSize); + Thread.sleep(1000); + + ThreadPoolMetrics afterResize = getThreadPoolMetrics(); + assertEquals(smallPoolSize, afterResize.maxThreads, "Max threads should reflect new size even under load"); + + executor.shutdown(); + Thread.sleep(4000); + + } finally { + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); + } + } +} From a8440a2587a3570016bca07e2f6f30165252b52b Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Wed, 22 Oct 2025 00:59:31 +0200 Subject: [PATCH 02/18] Thread pool monitoring - set number of threads directly from pool info --- .../v3/services/impl/monitor/ThreadPoolMonitor.java | 6 ++++++ .../impl/monitor/probes/ThreadPoolProbeProvider.java | 6 ++++++ .../impl/monitor/stats/ThreadPoolStatsProvider.java | 11 +++++++++++ 3 files changed, 23 insertions(+) diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java index d6f911f298b..bbb312ad934 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java @@ -66,6 +66,9 @@ public void onThreadAllocateEvent(AbstractThreadPool threadPool, Thread thread) grizzlyMonitoring.getThreadPoolProbeProvider().threadAllocatedEvent( monitoringId, threadPool.getConfig().getPoolName(), thread.getId()); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } @Override @@ -73,6 +76,9 @@ public void onThreadReleaseEvent(AbstractThreadPool threadPool, Thread thread) { grizzlyMonitoring.getThreadPoolProbeProvider().threadReleasedEvent( monitoringId, threadPool.getConfig().getPoolName(), thread.getId()); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } @Override diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java index 33eeb947bc4..4d6d4d75c70 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java @@ -76,4 +76,10 @@ public void threadReturnedToPoolEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, @ProbeParam("threadId") long threadId) {} + + @Probe(name="setCurrentThreadCountEvent") + public void setCurrentThreadCountEvent( + @ProbeParam("monitoringId") String monitoringId, + @ProbeParam("threadPoolName") String threadPoolName, + @ProbeParam("currentThreadCount") int currentThreadCount) {} } diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java index d20b4e19cff..4628bb49f2e 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java @@ -162,6 +162,17 @@ public void threadReturnedToPoolEvent( } } + @ProbeListener("glassfish:kernel:thread-pool:setCurrentThreadCountEvent") + public void setCurrentThreadCountEvent( + @ProbeParam("monitoringId") String monitoringId, + @ProbeParam("threadPoolName") String threadPoolName, + @ProbeParam("currentThreadCount") int currentThreadCount) { + + if (name.equals(monitoringId)) { + this.currentThreadCount.setCount(currentThreadCount); + } + } + @Reset public void reset() { if (threadPoolConfig != null) { From 01d1d598cfdc1e1d3699f285143826fd3881d19c Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Wed, 22 Oct 2025 02:26:31 +0200 Subject: [PATCH 03/18] More IT Tests to cover thread pool monitoring. --- .../monitoring/ThreadPoolMonitoringTest.java | 149 ++++++++++++++++-- 1 file changed, 134 insertions(+), 15 deletions(-) diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java index 142f6c37b1d..017fd7e5818 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java @@ -13,6 +13,7 @@ import java.io.File; import java.io.IOException; import java.net.HttpURLConnection; +import java.net.URL; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -283,25 +284,19 @@ private static File createDeployment() throws IOException { } private ThreadPoolMetrics getThreadPoolMetrics() { - // Try different possible monitoring paths - String[] possiblePaths = { - "server.network.http-listener-1.thread-pool", - "server.http-service.http-listener.http-listener-1.thread-pool", - "server.applications.application.threadpool-test.thread-pool", - "server.thread-pools.thread-pool.http-thread-pool" - }; - + return getThreadPoolMetrics("http-listener-1"); + } + + private ThreadPoolMetrics getThreadPoolMetrics(String listenerName) { // First, let's see what monitoring data is actually available AsadminResult listResult = ASADMIN.exec("list", "*thread*"); System.out.println("Available thread monitoring paths: " + listResult.getStdOut()); - // Try to get metrics from the most likely path - AsadminResult currentResult = ASADMIN.exec("get", "-m", - "server.network.http-listener-1.thread-pool.currentthreadcount"); - AsadminResult busyResult = ASADMIN.exec("get", "-m", - "server.network.http-listener-1.thread-pool.currentthreadsbusy"); - AsadminResult maxResult = ASADMIN.exec("get", "-m", - "server.network.http-listener-1.thread-pool.maxthreads"); + // Try to get metrics from the specified listener + String basePath = "server.network." + listenerName + ".thread-pool"; + AsadminResult currentResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadcount"); + AsadminResult busyResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadsbusy"); + AsadminResult maxResult = ASADMIN.exec("get", "-m", basePath + ".maxthreads"); return new ThreadPoolMetrics( extractValue(currentResult.getStdOut(), "currentthreadcount"), @@ -320,6 +315,20 @@ private int extractValue(String output, String metric) { return 0; } + private void makeRequest(String urlString) { + try { + URL url = new URL(urlString); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { + // Ignore connection errors for load testing + } + } + private static class ThreadPoolMetrics { final int currentThreadCount; final int currentThreadsBusy; @@ -470,6 +479,116 @@ void testThreadPoolSizeCycling() throws Exception { } } + @Test + void testAdminListenerThreadPoolMetrics() throws Exception { + // Get baseline metrics for admin-listener + ThreadPoolMetrics adminBaseline = getThreadPoolMetrics("admin-listener"); + + // Verify admin listener has valid baseline metrics + assertTrue(adminBaseline.currentThreadCount >= 0, "Admin listener should have valid thread count"); + assertTrue(adminBaseline.maxThreads > 0, "Admin listener should have positive max threads"); + assertTrue(adminBaseline.currentThreadsBusy >= 0, "Admin listener should have valid busy count"); + + // Create load on admin port (4848) using asadmin commands + ExecutorService executor = Executors.newFixedThreadPool(3); + + // Submit multiple concurrent admin requests + for (int i = 0; i < 5; i++) { + executor.submit(() -> { + try { + ASADMIN.exec("list", "applications"); + Thread.sleep(100); + ASADMIN.exec("get", "server.monitoring-service.*"); + } catch (Exception e) { + // Ignore for load testing + } + }); + } + + Thread.sleep(500); // Let requests start + + // Check metrics during admin load + ThreadPoolMetrics adminUnderLoad = getThreadPoolMetrics("admin-listener"); + + // Admin listener should maintain valid metrics under load + assertTrue(adminUnderLoad.currentThreadCount >= 0, "Admin listener should maintain valid thread count under load"); + assertTrue(adminUnderLoad.currentThreadsBusy <= adminUnderLoad.currentThreadCount, + "Admin listener: Busy threads should not exceed current threads"); + assertTrue(adminUnderLoad.maxThreads > 0, "Admin listener should maintain positive max threads"); + + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + + Thread.sleep(1000); // Let threads settle + + // Final metrics check + ThreadPoolMetrics adminFinal = getThreadPoolMetrics("admin-listener"); + assertTrue(adminFinal.currentThreadCount >= 0, "Admin listener final thread count should be valid"); + assertTrue(adminFinal.currentThreadsBusy >= 0, "Admin listener final busy count should be valid"); + } + + @Test + void testDualListenerThreadPoolMetrics() throws Exception { + // First create http-listener-2 if it doesn't exist + AsadminResult createResult = ASADMIN.exec("create-http-listener", + "--listenerport=8081", "--listeneraddress=0.0.0.0", + "--defaultvs=server", "http-listener-2"); + + try { + Thread.sleep(2000); // Allow listener to initialize + + // Get baseline metrics for both listeners + ThreadPoolMetrics listener1Baseline = getThreadPoolMetrics("http-listener-1"); + ThreadPoolMetrics listener2Baseline = getThreadPoolMetrics("http-listener-2"); + + // Verify both listeners have valid baseline metrics + assertTrue(listener1Baseline.currentThreadCount >= 0, "Listener 1 should have valid thread count"); + assertTrue(listener2Baseline.currentThreadCount >= 0, "Listener 2 should have valid thread count"); + + // Create load on both listeners simultaneously + ExecutorService executor = Executors.newFixedThreadPool(10); + + // Submit requests to both ports + for (int i = 0; i < 5; i++) { + executor.submit(() -> makeRequest("http://localhost:4848/" + APP_NAME + "/hello")); + executor.submit(() -> makeRequest("http://localhost:8081/" + APP_NAME + "/hello")); + } + + Thread.sleep(500); // Let requests start + + // Check metrics during dual load + ThreadPoolMetrics listener1UnderLoad = getThreadPoolMetrics("http-listener-1"); + ThreadPoolMetrics listener2UnderLoad = getThreadPoolMetrics("http-listener-2"); + + // Both listeners should show activity + assertTrue(listener1UnderLoad.currentThreadCount >= 0, "Listener 1 should maintain valid thread count under load"); + assertTrue(listener2UnderLoad.currentThreadCount >= 0, "Listener 2 should maintain valid thread count under load"); + + // Thread counts should be consistent with busy counts + assertTrue(listener1UnderLoad.currentThreadsBusy <= listener1UnderLoad.currentThreadCount, + "Listener 1: Busy threads should not exceed current threads"); + assertTrue(listener2UnderLoad.currentThreadsBusy <= listener2UnderLoad.currentThreadCount, + "Listener 2: Busy threads should not exceed current threads"); + + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + + Thread.sleep(1000); // Let threads settle + + // Final metrics check + ThreadPoolMetrics listener1Final = getThreadPoolMetrics("http-listener-1"); + ThreadPoolMetrics listener2Final = getThreadPoolMetrics("http-listener-2"); + + // Both should have valid final states + assertTrue(listener1Final.currentThreadCount >= 0, "Listener 1 final thread count should be valid"); + assertTrue(listener2Final.currentThreadCount >= 0, "Listener 2 final thread count should be valid"); + + } finally { + // Clean up http-listener-2 + ASADMIN.exec("delete-http-listener", "http-listener-2"); + } + } + @Test void testThreadPoolSizeUnderLoad() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); From e74233742cfeb6f61f936eb91fa929be07a7b520 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Fri, 24 Oct 2025 20:06:15 +0200 Subject: [PATCH 04/18] Set setCurrentThreadCountEvent directly --- .../impl/monitor/ThreadPoolMonitor.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java index bbb312ad934..7e5fdbe35ac 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java @@ -86,6 +86,9 @@ public void onMaxNumberOfThreadsEvent(AbstractThreadPool threadPool, int maxNumb grizzlyMonitoring.getThreadPoolProbeProvider().maxNumberOfThreadsReachedEvent( monitoringId, threadPool.getConfig().getPoolName(), maxNumberOfThreads); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } @Override @@ -95,6 +98,9 @@ public void onTaskDequeueEvent(AbstractThreadPool threadPool, Runnable task) { Thread.currentThread().getId()); grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskDequeuedEvent( monitoringId, task.getClass().getName()); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } @Override @@ -104,6 +110,9 @@ public void onTaskCancelEvent(AbstractThreadPool threadPool, Runnable task) { grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId()); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } @Override @@ -111,17 +120,26 @@ public void onTaskCompleteEvent(AbstractThreadPool threadPool, Runnable task) { grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId()); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } @Override public void onTaskQueueEvent(AbstractThreadPool threadPool, Runnable task) { grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskQueuedEvent( monitoringId, task.getClass().getName()); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } @Override public void onTaskQueueOverflowEvent(AbstractThreadPool threadPool) { grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskQueueOverflowEvent( monitoringId); + grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( + monitoringId, threadPool.getConfig().getPoolName(), + threadPool.getSize()); } } From 86a2a1815ef3cb3a2bd55d64500ae44121fb0217 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Fri, 24 Oct 2025 20:53:18 +0200 Subject: [PATCH 05/18] Fix monitoring in Embedded GlassFish Requires running with flashlight-agent on command line with -javaagent:flashlight-agent.jar When started with command "enable-monitoring --mbean true", metric will be available via JMX beans --- appserver/extras/embedded/all/pom.xml | 12 ++++ appserver/extras/embedded/pom.xml | 59 ++++++++++++++++++- appserver/extras/embedded/web/pom.xml | 12 ++++ .../flashlight/cli/EnableMonitoring.java | 2 +- .../impl/client/AgentAttacherInternal.java | 9 +++ 5 files changed, 92 insertions(+), 2 deletions(-) diff --git a/appserver/extras/embedded/all/pom.xml b/appserver/extras/embedded/all/pom.xml index 70bef2e780f..091b5c31168 100644 --- a/appserver/extras/embedded/all/pom.xml +++ b/appserver/extras/embedded/all/pom.xml @@ -127,6 +127,17 @@ + + org.codehaus.gmaven + groovy-maven-plugin + + + merge-manifest-values-to-properties + prepare-package + + + + maven-assembly-plugin @@ -168,6 +179,7 @@ org.glassfish.runnablejar.UberMain java.base/java.lang java.base/java.io java.base/java.util java.base/sun.nio.fs java.base/sun.net.www.protocol.jrt java.naming/javax.naming.spi java.rmi/sun.rmi.transport jdk.management/com.sun.management.internal java.base/jdk.internal.vm.annotation java.naming/com.sun.jndi.ldap java.base/jdk.internal.vm.annotation + ${probe.provider.class.names} diff --git a/appserver/extras/embedded/pom.xml b/appserver/extras/embedded/pom.xml index 0e265f7c36d..44395355722 100644 --- a/appserver/extras/embedded/pom.xml +++ b/appserver/extras/embedded/pom.xml @@ -16,7 +16,6 @@ SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 --> - 4.0.0 @@ -52,4 +51,62 @@ + + + + + org.codehaus.gmaven + groovy-maven-plugin + + + + merge-manifest-values-to-properties + + execute + + + + import java.util.jar.JarFile + import java.util.jar.Manifest + + def probeProviders = [] as Set + + project.artifacts.each { artifact -> + if (artifact.type == 'jar') { + try { + def jar = new JarFile(artifact.file) + def manifest = jar.manifest + if (manifest) { + def probeValue = manifest.mainAttributes.getValue('probe-provider-class-names') + if (probeValue) { + probeProviders.add(probeValue) + } + } + jar.close() + } catch (Exception e) { + // Ignore invalid JARs + } + } + } + + if (probeProviders) { + project.properties['probe.provider.class.names'] = probeProviders.join(',') + println "Found ${probeProviders.size()} probe providers in project dependencies" + } else { + println "No probe providers found in project dependencies" + } + + + + + + + + + diff --git a/appserver/extras/embedded/web/pom.xml b/appserver/extras/embedded/web/pom.xml index 515139dc483..c1f13f6bc07 100644 --- a/appserver/extras/embedded/web/pom.xml +++ b/appserver/extras/embedded/web/pom.xml @@ -113,6 +113,17 @@ + + org.codehaus.gmaven + groovy-maven-plugin + + + merge-manifest-values-to-properties + prepare-package + + + + maven-assembly-plugin @@ -192,6 +203,7 @@ org.glassfish.runnablejar.UberMain java.base/java.lang java.base/java.io java.base/java.util java.base/sun.nio.fs java.base/sun.net.www.protocol.jrt java.naming/javax.naming.spi java.rmi/sun.rmi.transport jdk.management/com.sun.management.internal java.base/jdk.internal.vm.annotation java.naming/com.sun.jndi.ldap java.base/jdk.internal.vm.annotation + ${probe.provider.class.names} diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java index 9d2bf7bc4b9..9b523048365 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java @@ -126,7 +126,7 @@ public void execute(AdminCommandContext context) { if (!AgentAttacher.attachAgent(pidInt, options)) { ActionReport.MessagePart part = report.getTopMessagePart().addChild(); part.setMessage(localStrings.getLocalString("attach.agent.exception", - "Can't attach the agent to the JVM.")); + "Can't attach the agent to the JVM. Restart with the flash-light agent attached by the -javaagent command line parameter")); report.setActionExitCode(ActionReport.ExitCode.FAILURE); return; } diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java index 56cdee4fd59..2c3ef7716ab 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java @@ -48,6 +48,7 @@ static boolean attachAgent() { return attachAgent(-1, ""); } + private static final String AGENT_CLASSNAME = "org.glassfish.flashlight.agent.ProbeAgentMain"; static boolean attachAgent(long pid, String options) { try { if (isAttached) { @@ -55,6 +56,14 @@ static boolean attachAgent(long pid, String options) { } if (pid < 0) { + + try { + ClassLoader.getSystemClassLoader().loadClass(AGENT_CLASSNAME); + isAttached = true; + return true; + } catch (Throwable t) { + } + pid = ProcessHandle.current().pid(); } From ba65ce2e456d86dcccf4e5a7bfa93507756f2b0b Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Fri, 24 Oct 2025 20:58:48 +0200 Subject: [PATCH 06/18] Increase log level printing command to execute in Embedded --- .../src/main/java/org/glassfish/runnablejar/UberMain.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nucleus/core/kernel/src/main/java/org/glassfish/runnablejar/UberMain.java b/nucleus/core/kernel/src/main/java/org/glassfish/runnablejar/UberMain.java index 7e78ffaf99c..675342b9240 100644 --- a/nucleus/core/kernel/src/main/java/org/glassfish/runnablejar/UberMain.java +++ b/nucleus/core/kernel/src/main/java/org/glassfish/runnablejar/UberMain.java @@ -40,6 +40,7 @@ import org.glassfish.runnablejar.commandline.CommandLineParser; import static java.lang.System.exit; +import static java.util.logging.Level.CONFIG; import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; @@ -176,7 +177,7 @@ private void runCommandPromptLoop() throws GlassFishException { } private void executeCommandFromString(String stringCommand) { - logger.log(FINE, () -> "Executing command: " + stringCommand); + logger.log(CONFIG, () -> "Executing command: " + stringCommand); // Split according to empty space but not if empty space is escaped by \ String[] split = stringCommand.split("(? Date: Fri, 24 Oct 2025 21:13:24 +0200 Subject: [PATCH 07/18] Add missing add-exports Needed for monitoring, at least in Embedded GlassFish. Otherwise: java.lang.reflect.InaccessibleObjectException: Unable to make public java.util.Enumeration jdk.internal.loader.BuiltinClassLoader.findResources(java.lang.String) throws java.io.IOException accessible: module java.base does not "exports jdk.internal.loader" to unnamed module @1de0aca6 at java.base/java.lang.reflect.AccessibleObject.throwInaccessibleObjectException(AccessibleObject.java:391) at java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:367) at java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:315) at java.base/java.lang.reflect.Method.checkCanSetAccessible(Method.java:203) at java.base/java.lang.reflect.Method.setAccessible(Method.java:197) at com.sun.enterprise.module.single.ManifestProxy.(ManifestProxy.java:71) at com.sun.enterprise.module.single.ProxyModuleDefinition.generate(ProxyModuleDefinition.java:146) at com.sun.enterprise.module.single.ProxyModuleDefinition.getManifest(ProxyModuleDefinition.java:134) at org.glassfish.admin.monitor.MonitoringBootstrap.addProvider(MonitoringBootstrap.java:298) --- .../admin/template/src/main/resources/config/domain.xml | 2 ++ appserver/extras/embedded/all/pom.xml | 4 ++-- appserver/extras/embedded/pom.xml | 5 +++++ .../embedded/shell/glassfish-embedded-static-shell/pom.xml | 6 +++--- appserver/extras/embedded/web/pom.xml | 4 ++-- nucleus/admin/template/src/main/resources/config/domain.xml | 2 ++ nucleus/core/kernel/src/test/resources/DomainTest.xml | 1 + 7 files changed, 17 insertions(+), 7 deletions(-) diff --git a/appserver/admin/template/src/main/resources/config/domain.xml b/appserver/admin/template/src/main/resources/config/domain.xml index c30fa534715..e1913bc4a58 100644 --- a/appserver/admin/template/src/main/resources/config/domain.xml +++ b/appserver/admin/template/src/main/resources/config/domain.xml @@ -223,6 +223,7 @@ --add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED --add-opens=java.base/jdk.internal.vm.annotation=ALL-UNNAMED + --add-exports=java.base/jdk.internal.loader=ALL-UNNAMED -Djdk.attach.allowAttachSelf=true @@ -421,6 +422,7 @@ --add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED --add-opens=java.base/jdk.internal.vm.annotation=ALL-UNNAMED + --add-exports=java.base/jdk.internal.loader=ALL-UNNAMED -Djdk.attach.allowAttachSelf=true diff --git a/appserver/extras/embedded/all/pom.xml b/appserver/extras/embedded/all/pom.xml index 091b5c31168..ffd5d797b10 100644 --- a/appserver/extras/embedded/all/pom.xml +++ b/appserver/extras/embedded/all/pom.xml @@ -177,8 +177,8 @@ true org.glassfish.main.embedded.all org.glassfish.runnablejar.UberMain - java.base/java.lang java.base/java.io java.base/java.util java.base/sun.nio.fs java.base/sun.net.www.protocol.jrt java.naming/javax.naming.spi java.rmi/sun.rmi.transport jdk.management/com.sun.management.internal java.base/jdk.internal.vm.annotation - java.naming/com.sun.jndi.ldap java.base/jdk.internal.vm.annotation + ${glassfish.embedded.add-opens} + ${glassfish.embedded.add-exports} ${probe.provider.class.names} diff --git a/appserver/extras/embedded/pom.xml b/appserver/extras/embedded/pom.xml index 44395355722..0902b2f5aa0 100644 --- a/appserver/extras/embedded/pom.xml +++ b/appserver/extras/embedded/pom.xml @@ -29,6 +29,11 @@ pom GlassFish Embedded modules + + + java.base/java.lang java.base/java.io java.base/java.util java.base/sun.nio.fs java.base/sun.net.www.protocol.jrt java.naming/javax.naming.spi java.rmi/sun.rmi.transport jdk.management/com.sun.management.internal java.base/jdk.internal.vm.annotation + java.naming/com.sun.jndi.ldap java.base/jdk.internal.vm.annotation java.base/jdk.internal.loader + common diff --git a/appserver/extras/embedded/shell/glassfish-embedded-static-shell/pom.xml b/appserver/extras/embedded/shell/glassfish-embedded-static-shell/pom.xml index 59d9311c990..2bc22191d43 100755 --- a/appserver/extras/embedded/shell/glassfish-embedded-static-shell/pom.xml +++ b/appserver/extras/embedded/shell/glassfish-embedded-static-shell/pom.xml @@ -1,7 +1,7 @@ -Djdk.attach.allowAttachSelf=true @@ -342,6 +343,7 @@ --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED + --add-exports=java.base/jdk.internal.loader=ALL-UNNAMED -Djdk.attach.allowAttachSelf=true diff --git a/nucleus/core/kernel/src/test/resources/DomainTest.xml b/nucleus/core/kernel/src/test/resources/DomainTest.xml index 82a37768268..28bb9b5b693 100644 --- a/nucleus/core/kernel/src/test/resources/DomainTest.xml +++ b/nucleus/core/kernel/src/test/resources/DomainTest.xml @@ -151,6 +151,7 @@ --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED --add-opens=java.naming/javax.naming.spi=org.glassfish.main.jdke --add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED + --add-exports=java.base/jdk.internal.loader=ALL-UNNAMED From 9ce13248b7b27fce2fdf75b1cdcd50fbe092cad6 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Sat, 25 Oct 2025 15:12:03 +0200 Subject: [PATCH 08/18] Refactor flashlight-agent attachment code to simplify it. --- .../flashlight/FlashlightLoggerInfo.java | 5 -- .../flashlight/cli/EnableMonitoring.java | 2 +- .../flashlight/impl/client/AgentAttacher.java | 60 +++++++++++++++---- .../impl/client/AgentAttacherInternal.java | 5 +- .../ProbeProviderClassFileTransformer.java | 27 +-------- 5 files changed, 52 insertions(+), 47 deletions(-) diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/FlashlightLoggerInfo.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/FlashlightLoggerInfo.java index e261db84164..492a7faa018 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/FlashlightLoggerInfo.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/FlashlightLoggerInfo.java @@ -132,11 +132,6 @@ public static Logger getLogger() { level = "WARNING") public static final String NO_ATTACH_API = LOGMSG_PREFIX + "-00510"; - @LogMessageInfo( - message = "Error while getting Instrumentation object from ProbeAgentMain", - level = "WARNING") - public static final String NO_ATTACH_GET = LOGMSG_PREFIX + "-00511"; - @LogMessageInfo( message = "DTrace is not available.", cause="This is caused if following are missing: \n1. JDK 7 is required to run DTrace\n2. glassfish-dtrace.jar value-add is required for DTrace", diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java index 9b523048365..e7300b1dae0 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/cli/EnableMonitoring.java @@ -126,7 +126,7 @@ public void execute(AdminCommandContext context) { if (!AgentAttacher.attachAgent(pidInt, options)) { ActionReport.MessagePart part = report.getTopMessagePart().addChild(); part.setMessage(localStrings.getLocalString("attach.agent.exception", - "Can't attach the agent to the JVM. Restart with the flash-light agent attached by the -javaagent command line parameter")); + "Can't attach the monitoring agent to the JVM. Restart with the flashlight-agent.jar attached by the -javaagent command line parameter")); report.setActionExitCode(ActionReport.ExitCode.FAILURE); return; } diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacher.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacher.java index 8cf08ae96b5..d080c203a08 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacher.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacher.java @@ -13,26 +13,63 @@ * * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 */ - package org.glassfish.flashlight.impl.client; +import java.lang.instrument.Instrumentation; +import java.lang.reflect.Method; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.glassfish.flashlight.FlashlightLoggerInfo; + + /** * created May 26, 2011 + * * @author Byron Nevins */ public final class AgentAttacher { + + private static final String AGENT_CLASSNAME = "org.glassfish.flashlight.agent.ProbeAgentMain"; + + private static final Logger logger = FlashlightLoggerInfo.getLogger(); + + public static Optional getInstrumentation() { + try { + Class agentMainClass = null; + ClassLoader classLoader = ClassLoader.getSystemClassLoader(); + + try { + agentMainClass = classLoader.loadClass(AGENT_CLASSNAME); + } catch (Throwable t) { + // need throwable, not Exception - it may throw an Error! + // try one more time after attempting to attach. + AgentAttacher.attachAgent(); + // might throw + agentMainClass = classLoader.loadClass(AGENT_CLASSNAME); + } + + Method mthd = agentMainClass.getMethod("getInstrumentation", null); + return Optional.ofNullable((Instrumentation) mthd.invoke(null, null)); + } catch (Throwable throwable) { + logger.log(Level.WARNING, "Error while getting Instrumentation object from ProbeAgentMain", throwable); + return Optional.empty(); + } + } + public synchronized static boolean canAttach() { return canAttach; } public synchronized static boolean isAttached() { try { - if (!canAttach) + if (!canAttach) { return false; + } return AgentAttacherInternal.isAttached(); - } - catch (Throwable t) { + } catch (Throwable t) { return false; } } @@ -40,24 +77,24 @@ public synchronized static boolean isAttached() { public synchronized static boolean attachAgent() { try { - if (!canAttach) + if (!canAttach) { return false; + } return attachAgent(-1, ""); - } - catch (Throwable t) { + } catch (Throwable t) { return false; } } public synchronized static boolean attachAgent(int pid, String options) { try { - if (!canAttach) + if (!canAttach) { return false; + } return AgentAttacherInternal.attachAgent(pid, options); - } - catch (Throwable t) { + } catch (Throwable t) { return false; } } @@ -71,8 +108,7 @@ public synchronized static boolean attachAgent(int pid, String options) { // this is a distinct possibility in embedded mode. AgentAttacherInternal.isAttached(); b = true; - } - catch (Throwable t) { + } catch (Throwable t) { b = false; } canAttach = b; diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java index 2c3ef7716ab..2b395d0e068 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/AgentAttacherInternal.java @@ -48,7 +48,6 @@ static boolean attachAgent() { return attachAgent(-1, ""); } - private static final String AGENT_CLASSNAME = "org.glassfish.flashlight.agent.ProbeAgentMain"; static boolean attachAgent(long pid, String options) { try { if (isAttached) { @@ -57,11 +56,9 @@ static boolean attachAgent(long pid, String options) { if (pid < 0) { - try { - ClassLoader.getSystemClassLoader().loadClass(AGENT_CLASSNAME); + if (AgentAttacher.getInstrumentation().isPresent()) { isAttached = true; return true; - } catch (Throwable t) { } pid = ProcessHandle.current().pid(); diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/ProbeProviderClassFileTransformer.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/ProbeProviderClassFileTransformer.java index ba5c9383c66..cbb1c219e23 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/ProbeProviderClassFileTransformer.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/impl/client/ProbeProviderClassFileTransformer.java @@ -48,7 +48,6 @@ import static org.glassfish.embeddable.GlassFishVariable.INSTALL_ROOT; import static org.glassfish.flashlight.FlashlightLoggerInfo.NO_ATTACH_API; -import static org.glassfish.flashlight.FlashlightLoggerInfo.NO_ATTACH_GET; import static org.glassfish.flashlight.FlashlightLoggerInfo.REGISTRATION_ERROR; import static org.glassfish.flashlight.FlashlightLoggerInfo.RETRANSFORMATION_ERROR; import static org.glassfish.flashlight.FlashlightLoggerInfo.WRITE_ERROR; @@ -82,7 +81,6 @@ public class ProbeProviderClassFileTransformer implements ClassFileTransformer { private static final Instrumentation instrumentation; private static boolean _debug = Boolean.parseBoolean(Utility.getEnvOrProp("AS_DEBUG")); private static boolean emittedAttachUnavailableMessageAlready = false; - private static final String AGENT_CLASSNAME = "org.glassfish.flashlight.agent.ProbeAgentMain"; private static final Logger logger = FlashlightLoggerInfo.getLogger(); private ProbeProviderClassFileTransformer(Class providerClass) { @@ -494,28 +492,7 @@ private Method getMethod(FlashlightProbe probe) throws NoSuchMethodException { if (AgentAttacher.canAttach()) { canAttach = true; - try { - ClassLoader classLoader = ClassLoader.getSystemClassLoader(); - - try { - agentMainClass = classLoader.loadClass(AGENT_CLASSNAME); - } - catch (Throwable t) { - // need throwable, not Exception - it may throw an Error! - // try one more time after attempting to attach. - AgentAttacher.attachAgent(); - // might throw - agentMainClass = classLoader.loadClass(AGENT_CLASSNAME); - } - - Method mthd = agentMainClass.getMethod("getInstrumentation", null); - nonFinalInstrumentation = (Instrumentation) mthd.invoke(null, null); - } - catch (Throwable t) { - nonFinalInstrumentation = null; - // save it for nice neat message code below - throwable = t; - } + nonFinalInstrumentation = AgentAttacher.getInstrumentation().orElse(null); } // set the final instrumentation = nonFinalInstrumentation; @@ -525,7 +502,7 @@ private Method getMethod(FlashlightProbe probe) throws NoSuchMethodException { } else if (instrumentation != null) { Log.info("yes.attach.api", instrumentation); } else { - logger.log(Level.WARNING, NO_ATTACH_GET, throwable); + logger.log(Level.WARNING, "Could not attach agent"); } } } From da7c94145ed627115c1824789ee01842d4ad8161 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Sat, 25 Oct 2025 15:12:25 +0200 Subject: [PATCH 09/18] Document monitoring for Embedded GlassFish --- .../main/asciidoc/embedded-server-guide.adoc | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc b/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc index f7d9aa57dc7..05f41261739 100644 --- a/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc +++ b/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc @@ -20,6 +20,7 @@ The following topics are addressed here: * xref:#testing-applications-with-the-maven-plug-in-for-embedded-glassfish-server[Testing Applications with the Maven Plug-in for Embedded {productName}] * xref:#GSESG00039[Using the EJB 3.1 Embeddable API with Embedded {productName}] * xref:#changing-log-levels-in-embedded-glassfish-server[Changing Log Levels in Embedded {productName}] +* xref:#monitoring-embedded-glassfish-server-with-jmx[Monitoring Embedded {productName} with JMX] * xref:#default-java-persistence-data-source-for-embedded-glassfish-server[Default Java Persistence Data Source for Embedded {productName}] * xref:#restrictions-for-embedded-glassfish-server[Restrictions for Embedded {productName}] @@ -2061,6 +2062,93 @@ command when you invoke Embedded {productName}. For example: java -Djava.util.logging.config.file=customlogging.properties MyEmbeddedGlassFish ---- +[[monitoring-embedded-glassfish-server-with-jmx]] + +== Monitoring Embedded {productName} with JMX + +Embedded {productName} supports monitoring through JMX MBeans, similar to regular {productName} Server. However, you must attache the flashlight agent on command line to enable monitoring functionality. + +=== Prerequisites + +To enable monitoring in Embedded {productName}, you need the flashlight agent JAR file: + +* Download from Maven Central: `org.glassfish.main.flashlight:flashlight-agent` +* Or use from an existing {productName} installation: `glassfish/lib/monitor/flashlight-agent.jar` + +=== Enabling Monitoring Modules + +By default, the monitoring service and MBeans support are enabled, but no monitoring modules are active. You can enable specific monitoring modules using one of two methods: + +==== Method 1: Command Line + +Start Embedded {productName} with the `enable-monitoring` command: + +[source] +---- +java -javaagent:/path/to/flashlight-agent.jar -jar glassfish-embedded-all.jar 'enable-monitoring --modules thread-pool:http-service' +---- + +==== Method 2: Properties File + +Create a `glassfish.properties` file in the current directory with monitoring configuration: + +[source] +---- +configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH +configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=HIGH +---- + +Then start Embedded {productName}: + +[source] +---- +java -javaagent:/path/to/flashlight-agent.jar -jar glassfish-embedded-all.jar +---- + +=== Accessing Monitoring Data + +Once monitoring modules are enabled, you can access monitoring data through JMX clients such as VisualVM or JConsole. + +==== Using VisualVM + +1. Connect to the Embedded {productName} process in VisualVM +2. Install the MBeans extension if not already available +3. Navigate to the MBeans tab +4. Execute the `bootAMX` operation first (this is required, same as with regular {productName} Server) +5. Monitoring data will be available under the `amx` node, in nodes that end with `-mon`, such as `thread-pool-mon` or `request-mon` + +==== Exposing JMX on a Specific Port + +To allow remote JMX connections, you can expose the JMX server on a specific port by adding JVM system properties: + +[source] +---- +java -javaagent:/path/to/flashlight-agent.jar \ + -Dcom.sun.management.jmxremote \ + -Dcom.sun.management.jmxremote.port=8686 \ + -Dcom.sun.management.jmxremote.authenticate=false \ + -Dcom.sun.management.jmxremote.ssl=false \ + -jar glassfish-embedded-all.jar 'enable-monitoring --modules thread-pool:http-service' +---- + +[WARNING] +==== +Exposing JMX without authentication and SSL is not secure and should only be used in development environments. JMX MBeans include management beans that allow executing administrative operations on {productName}, which could be exploited by unauthorized users. In production environments, always enable authentication and SSL for JMX connections. +==== + +==== Available Monitoring Modules + +Embedded {productName} supports the same monitoring modules as regular {productName} Server, including: + +* `thread-pool` - Thread pool statistics +* `http-service` - HTTP service metrics +* `web-container` - Web container statistics +* `ejb-container` - EJB container metrics +* `transaction-service` - Transaction service data +* `jvm` - JVM statistics + +For comprehensive information about available monitoring modules and their metrics, see xref:administration-guide.adoc#administering-the-monitoring-service[Administering the Monitoring Service] in the {productName} Administration Guide. + [[default-java-persistence-data-source-for-embedded-glassfish-server]] == Default Java Persistence Data Source for Embedded {productName} From be9906c39a215579fa758df1857d5b0c60838d33 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Sat, 25 Oct 2025 16:34:03 +0200 Subject: [PATCH 10/18] Test for monitoring of Embedded GlassFish via JMX --- appserver/tests/embedded/runnable/pom.xml | 5 + .../embedded/runnable/MonitoringTest.java | 160 ++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java diff --git a/appserver/tests/embedded/runnable/pom.xml b/appserver/tests/embedded/runnable/pom.xml index 137b8e3bd6c..79955d4eac4 100644 --- a/appserver/tests/embedded/runnable/pom.xml +++ b/appserver/tests/embedded/runnable/pom.xml @@ -51,6 +51,11 @@ org.glassfish.main.extras glassfish-embedded-web + + org.glassfish.main.flashlight + flashlight-agent + ${project.version} + diff --git a/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java new file mode 100644 index 00000000000..e30b10f6130 --- /dev/null +++ b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ +package org.glassfish.tests.embedded.runnable; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import javax.management.MBeanServerConnection; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; + +import org.glassfish.tests.embedded.runnable.TestArgumentProviders.GfEmbeddedJarNameProvider; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsSource; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author Ondro Mihalyi + */ +public class MonitoringTest { + + private static final Logger LOG = Logger.getLogger(MonitoringTest.class.getName()); + private static final int JMX_PORT = 8686; + private static final int WAIT_SECONDS = 30; + + @ParameterizedTest + @ArgumentsSource(GfEmbeddedJarNameProvider.class) + void testJmxMonitoringWithFlashlightAgent(String gfEmbeddedJarName) throws Exception { + Process gfEmbeddedProcess = null; + JMXConnector jmxConnector = null; + try { + gfEmbeddedProcess = startGlassFishWithJmx(gfEmbeddedJarName); + + jmxConnector = connectToJmx(); + MBeanServerConnection mbsc = jmxConnector.getMBeanServerConnection(); + + bootAmxAndWait(mbsc); + + verifyMonitoringMBeans(mbsc); + + } finally { + cleanup(jmxConnector, gfEmbeddedProcess); + } + } + + private File createMonitoringPropertiesFile() throws Exception { + File propertiesFile = File.createTempFile("monitoring", ".properties"); + java.nio.file.Files.write(propertiesFile.toPath(), List.of( + "configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=HIGH", + "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH" + )); + return propertiesFile; + } + + private Process startGlassFishWithJmx(String gfEmbeddedJarName) throws IOException { + List arguments = new ArrayList<>(); + arguments.add(ProcessHandle.current().info().command().get()); + arguments.addAll(List.of( + "-javaagent:flashlight-agent.jar", + "-Dcom.sun.management.jmxremote", + "-Dcom.sun.management.jmxremote.port=" + JMX_PORT, + "-Dcom.sun.management.jmxremote.authenticate=false", + "-Dcom.sun.management.jmxremote.ssl=false", + "-jar", gfEmbeddedJarName, + "--noPort", + "enable-monitoring --modules http-service" + )); + + return new ProcessBuilder() + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .redirectError(ProcessBuilder.Redirect.PIPE) + .command(arguments) + .start(); + } + + private JMXConnector connectToJmx() throws Exception { + JMXServiceURL serviceURL = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://localhost:" + JMX_PORT + "/jmxrmi"); + + for (int i = 0; i < WAIT_SECONDS * 2; i++) { + try { + return JMXConnectorFactory.connect(serviceURL, null); + } catch (Exception e) { + Thread.sleep(500); + } + } + throw new IllegalStateException("Could not connect to JMX in " + WAIT_SECONDS + " seconds"); + } + + private void bootAmxAndWait(MBeanServerConnection mbsc) throws Exception { + ObjectName bootAMXObjectName = new ObjectName("amx-support:type=boot-amx"); + + for (int i = 0; i < WAIT_SECONDS * 2; i++) { + if (mbsc.isRegistered(bootAMXObjectName)) { + break; + } + Thread.sleep(500); + } + + + assertTrue(mbsc.isRegistered(bootAMXObjectName), "bootAMX is registered"); + + mbsc.invoke(bootAMXObjectName, "bootAMX", null, null); + + // Wait for AMX runtime to be available + for (int i = 0; i < WAIT_SECONDS * 2; i++) { + Set runtimeBeans = mbsc.queryNames(new ObjectName("amx:pp=/,type=runtime"), null); + if (!runtimeBeans.isEmpty()) { + return; + } + Thread.sleep(500); + } + throw new IllegalStateException("AMX runtime not available after " + WAIT_SECONDS + " seconds"); + } + + private void verifyMonitoringMBeans(MBeanServerConnection mbsc) throws Exception { + Set requestBeans = mbsc.queryNames(new ObjectName("amx:type=request-mon,*"), null); + assertTrue(!requestBeans.isEmpty(), "Request monitoring MBean should be present"); + + // Verify we can read monitoring data + ObjectName requestBean = requestBeans.iterator().next(); + assertNotNull(mbsc.getAttribute(requestBean, "countrequests"), "Should be able to read request count"); + } + + private void cleanup(JMXConnector jmxConnector, Process process) throws InterruptedException { + if (jmxConnector != null) try { jmxConnector.close(); } catch (Exception ignored) {} + if (process != null && process.isAlive()) { + process.destroyForcibly(); + process.waitFor(5, TimeUnit.SECONDS); + } + } + + private void cleanupFiles(File... files) { + for (File file : files) { + Optional.ofNullable(file).ifPresent(File::delete); + } + } +} From 0a14238b0a9085c83283e375ac183c968a3d2ad5 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Sat, 25 Oct 2025 20:55:57 +0200 Subject: [PATCH 11/18] Test improvement - connect to JMX via socket instead of port Disable test for Embedded Web - it doesn't support AMX because it doesn't include glassfish-mbeanserver artifact. --- .../embedded/runnable/GfEmbeddedUtils.java | 13 +- .../embedded/runnable/MonitoringTest.java | 142 ++++++++++-------- .../tests/embedded/runnable/TestUtils.java | 47 ++++++ .../main/asciidoc/embedded-server-guide.adoc | 5 + 4 files changed, 138 insertions(+), 69 deletions(-) create mode 100644 appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/TestUtils.java diff --git a/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/GfEmbeddedUtils.java b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/GfEmbeddedUtils.java index 3933a45a1b3..8e779508b7a 100644 --- a/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/GfEmbeddedUtils.java +++ b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/GfEmbeddedUtils.java @@ -53,15 +53,22 @@ public static Process runGlassFishEmbedded(String glassfishEmbeddedJarName, Stri return runGlassFishEmbedded(glassfishEmbeddedJarName, List.of(), additionalArguments); } - public static Process runGlassFishEmbedded(String glassfishEmbeddedJarName, List jvmOpts, String... additionalArguments) throws IOException { + public static Process runGlassFishEmbedded(String glassfishEmbeddedJarName, List jvmOpts, String... +additionalArguments) throws IOException { + return runGlassFishEmbedded(glassfishEmbeddedJarName, false, jvmOpts, additionalArguments); + } + + public static Process runGlassFishEmbedded(String glassfishEmbeddedJarName, boolean keepRunning, List jvmOpts, String... additionalArguments) throws IOException { List arguments = new ArrayList<>(); arguments.add(ProcessHandle.current().info().command().get()); addDebugArgsIfDebugEnabled(arguments); arguments.addAll(jvmOpts); arguments.addAll(List.of( "-jar", glassfishEmbeddedJarName, - "--noPort", - "--stop")); + "--noPort")); + if (!keepRunning) { + arguments.add("--stop"); + } for (String argument : additionalArguments) { arguments.add(argument); } diff --git a/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java index e30b10f6130..c6b142ba134 100644 --- a/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java +++ b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/MonitoringTest.java @@ -15,15 +15,19 @@ */ package org.glassfish.tests.embedded.runnable; +import com.sun.tools.attach.VirtualMachine; +import com.sun.tools.attach.VirtualMachineDescriptor; + import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import javax.management.JMException; import javax.management.MBeanServerConnection; import javax.management.ObjectName; import javax.management.remote.JMXConnector; @@ -31,11 +35,18 @@ import javax.management.remote.JMXServiceURL; import org.glassfish.tests.embedded.runnable.TestArgumentProviders.GfEmbeddedJarNameProvider; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.exporter.ZipExporter; +import org.jboss.shrinkwrap.api.spec.WebArchive; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; +import static java.util.logging.Level.WARNING; +import static org.glassfish.tests.embedded.runnable.GfEmbeddedUtils.runGlassFishEmbedded; +import static org.glassfish.tests.embedded.runnable.ShrinkwrapUtils.logArchiveContent; +import static org.glassfish.tests.embedded.runnable.TestUtils.waitFor; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * @author Ondro Mihalyi @@ -43,113 +54,112 @@ public class MonitoringTest { private static final Logger LOG = Logger.getLogger(MonitoringTest.class.getName()); - private static final int JMX_PORT = 8686; - private static final int WAIT_SECONDS = 30; @ParameterizedTest @ArgumentsSource(GfEmbeddedJarNameProvider.class) void testJmxMonitoringWithFlashlightAgent(String gfEmbeddedJarName) throws Exception { + assumeTrue(!gfEmbeddedJarName.endsWith("web.jar"), + "AMX is not supported by glassfish-embedded-web.jar, skipping this test scenario"); Process gfEmbeddedProcess = null; JMXConnector jmxConnector = null; + File warFile = null; try { - gfEmbeddedProcess = startGlassFishWithJmx(gfEmbeddedJarName); + // an app needs to be deployed to initialize request monitoring + warFile = createEmptyApp(); + gfEmbeddedProcess = startGlassFishWithJmx(gfEmbeddedJarName, warFile); jmxConnector = connectToJmx(); MBeanServerConnection mbsc = jmxConnector.getMBeanServerConnection(); - bootAmxAndWait(mbsc); + bootAmx(mbsc); verifyMonitoringMBeans(mbsc); } finally { - cleanup(jmxConnector, gfEmbeddedProcess); + cleanup(jmxConnector, gfEmbeddedProcess, warFile); } } - private File createMonitoringPropertiesFile() throws Exception { - File propertiesFile = File.createTempFile("monitoring", ".properties"); - java.nio.file.Files.write(propertiesFile.toPath(), List.of( - "configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=HIGH", - "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH" - )); - return propertiesFile; - } + private File createEmptyApp() throws Exception { + WebArchive war = ShrinkWrap.create(WebArchive.class, "empty-app.war") + .addAsWebInfResource("", "beans.xml"); - private Process startGlassFishWithJmx(String gfEmbeddedJarName) throws IOException { - List arguments = new ArrayList<>(); - arguments.add(ProcessHandle.current().info().command().get()); - arguments.addAll(List.of( - "-javaagent:flashlight-agent.jar", - "-Dcom.sun.management.jmxremote", - "-Dcom.sun.management.jmxremote.port=" + JMX_PORT, - "-Dcom.sun.management.jmxremote.authenticate=false", - "-Dcom.sun.management.jmxremote.ssl=false", - "-jar", gfEmbeddedJarName, - "--noPort", - "enable-monitoring --modules http-service" - )); - - return new ProcessBuilder() - .redirectOutput(ProcessBuilder.Redirect.PIPE) - .redirectError(ProcessBuilder.Redirect.PIPE) - .command(arguments) - .start(); + File warFile = File.createTempFile("empty-app", ".war"); + war.as(ZipExporter.class).exportTo(warFile, true); + logArchiveContent(war, "empty-app.war", LOG::info); + return warFile; } - private JMXConnector connectToJmx() throws Exception { - JMXServiceURL serviceURL = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://localhost:" + JMX_PORT + "/jmxrmi"); + private Process startGlassFishWithJmx(String gfEmbeddedJarName, File warFile) throws IOException { + return runGlassFishEmbedded(gfEmbeddedJarName, true, + List.of("-Dcom.sun.management.jmxremote", + "-javaagent:flashlight-agent.jar"), + "enable-monitoring --modules http-service", + warFile.getAbsolutePath()); + } - for (int i = 0; i < WAIT_SECONDS * 2; i++) { + private JMXConnector connectToJmx() throws InterruptedException { + return waitFor("JMX connector", () -> { + VirtualMachine vm = null; + String connectorAddress = null; try { - return JMXConnectorFactory.connect(serviceURL, null); + // Find GlassFish process by looking for our jar + for (VirtualMachineDescriptor vmd : VirtualMachine.list()) { + if (vmd.displayName().contains("glassfish-embedded")) { + vm = VirtualMachine.attach(vmd.id()); + + // Get or create JMX connector address + Properties props = vm.getAgentProperties(); + vm.detach(); + vm = null; + connectorAddress = props.getProperty("com.sun.management.jmxremote.localConnectorAddress"); + + if (connectorAddress != null) { + JMXServiceURL serviceURL = new JMXServiceURL(connectorAddress); + return JMXConnectorFactory.connect(serviceURL, null); + } else { + throw new UnsupportedOperationException("Connector address not available!"); + } + } + } } catch (Exception e) { - Thread.sleep(500); + LOG.log(WARNING, e.getMessage(), e); + if (vm != null) try { vm.detach(); } catch (Exception ignored) {} } - } - throw new IllegalStateException("Could not connect to JMX in " + WAIT_SECONDS + " seconds"); + return (JMXConnector)null; + }); } - private void bootAmxAndWait(MBeanServerConnection mbsc) throws Exception { + private void bootAmx(MBeanServerConnection mbsc) throws Exception { ObjectName bootAMXObjectName = new ObjectName("amx-support:type=boot-amx"); - for (int i = 0; i < WAIT_SECONDS * 2; i++) { - if (mbsc.isRegistered(bootAMXObjectName)) { - break; - } - Thread.sleep(500); - } - - - assertTrue(mbsc.isRegistered(bootAMXObjectName), "bootAMX is registered"); + waitFor("bootAMX", () -> mbsc.isRegistered(bootAMXObjectName) ? true : null); mbsc.invoke(bootAMXObjectName, "bootAMX", null, null); - - // Wait for AMX runtime to be available - for (int i = 0; i < WAIT_SECONDS * 2; i++) { - Set runtimeBeans = mbsc.queryNames(new ObjectName("amx:pp=/,type=runtime"), null); - if (!runtimeBeans.isEmpty()) { - return; - } - Thread.sleep(500); - } - throw new IllegalStateException("AMX runtime not available after " + WAIT_SECONDS + " seconds"); } - private void verifyMonitoringMBeans(MBeanServerConnection mbsc) throws Exception { - Set requestBeans = mbsc.queryNames(new ObjectName("amx:type=request-mon,*"), null); - assertTrue(!requestBeans.isEmpty(), "Request monitoring MBean should be present"); + private void verifyMonitoringMBeans(final MBeanServerConnection mbsc) throws InterruptedException, IOException, JMException { + Set requestBeans = waitFor("equest-mon bean", () -> { + Set result = mbsc.queryNames(new ObjectName("amx:type=request-mon,*"), null); + return result.isEmpty() ? null : result; + }); // Verify we can read monitoring data ObjectName requestBean = requestBeans.iterator().next(); assertNotNull(mbsc.getAttribute(requestBean, "countrequests"), "Should be able to read request count"); } - private void cleanup(JMXConnector jmxConnector, Process process) throws InterruptedException { + private void cleanup(JMXConnector jmxConnector, Process process, File... files) throws InterruptedException { if (jmxConnector != null) try { jmxConnector.close(); } catch (Exception ignored) {} if (process != null && process.isAlive()) { - process.destroyForcibly(); + process.destroy(); process.waitFor(5, TimeUnit.SECONDS); + if (process.isAlive()) { + process.destroy(); + process.waitFor(5, TimeUnit.SECONDS); + } } + cleanupFiles(files); } private void cleanupFiles(File... files) { diff --git a/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/TestUtils.java b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/TestUtils.java new file mode 100644 index 00000000000..394e1eb66bf --- /dev/null +++ b/appserver/tests/embedded/runnable/src/test/java/org/glassfish/tests/embedded/runnable/TestUtils.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ +package org.glassfish.tests.embedded.runnable; + +/** + * + * @author Ondro Mihalyi + */ +public abstract class TestUtils { + + private static final int WAIT_SECONDS = 30; + + private TestUtils() { + } + + public static T waitFor(String what, org.junit.jupiter.api.function.ThrowingSupplier supplier) throws InterruptedException { + Throwable lastThrowable = null; + for (int i = 0; i < WAIT_SECONDS * 2; i++) { + try { + T result = supplier.get(); + if (result != null) { + return result; + } + } catch (UnsupportedOperationException unrecoverableError) { + throw unrecoverableError; + } catch (Throwable ignore) { + lastThrowable = ignore; + } + Thread.sleep(500); + } + throw new RuntimeException(what + " not received within timeout", lastThrowable); + } + +} diff --git a/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc b/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc index 05f41261739..8d495277d5b 100644 --- a/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc +++ b/docs/embedded-server-guide/src/main/asciidoc/embedded-server-guide.adoc @@ -2068,6 +2068,11 @@ java -Djava.util.logging.config.file=customlogging.properties MyEmbeddedGlassFis Embedded {productName} supports monitoring through JMX MBeans, similar to regular {productName} Server. However, you must attache the flashlight agent on command line to enable monitoring functionality. +[IMPORTANT] +==== +AMX (Application Management Extensions) monitoring is only available in `glassfish-embedded-all.jar`. The `glassfish-embedded-web.jar` does not include AMX support, which limits the available monitoring MBeans. +==== + === Prerequisites To enable monitoring in Embedded {productName}, you need the flashlight agent JAR file: From 191591a183aaae26252e255b6bc107b762db273f Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Sun, 26 Oct 2025 16:57:34 +0100 Subject: [PATCH 12/18] Always use Hamcrest assertions --- .../monitoring/ThreadPoolMonitoringTest.java | 334 +++++++++--------- 1 file changed, 159 insertions(+), 175 deletions(-) diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java index 017fd7e5818..40620dcf623 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java @@ -31,10 +31,10 @@ import org.junit.jupiter.api.Test; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; public class ThreadPoolMonitoringTest { @@ -47,7 +47,7 @@ static void deployApp() throws IOException { try { AsadminResult result = ASADMIN.exec("deploy", warFile.getAbsolutePath()); assertThat(result, AsadminResultMatcher.asadminOK()); - + // Enable monitoring ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH"); ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.http-service=HIGH"); @@ -69,14 +69,14 @@ static void undeployApp() { void testThreadPoolMetricsUnderLoad() throws Exception { // Get baseline metrics ThreadPoolMetrics baseline = getThreadPoolMetrics(); - assertTrue(baseline.currentThreadCount >= 0); - assertTrue(baseline.currentThreadsBusy >= 0); - assertTrue(baseline.currentThreadsBusy <= baseline.currentThreadCount); + assertThat("baseline current thread count", baseline.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("baseline busy threads", baseline.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("baseline busy threads", baseline.currentThreadsBusy, lessThanOrEqualTo(baseline.currentThreadCount)); // Generate concurrent load ExecutorService executor = Executors.newFixedThreadPool(10); CompletableFuture[] futures = new CompletableFuture[20]; - + for (int i = 0; i < futures.length; i++) { futures[i] = CompletableFuture.runAsync(() -> { try { @@ -93,15 +93,11 @@ void testThreadPoolMetricsUnderLoad() throws Exception { // Check metrics during load Thread.sleep(100); // Let requests start ThreadPoolMetrics duringLoad = getThreadPoolMetrics(); - - assertThat("Current threads should be >= baseline", - duringLoad.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); - assertThat("Busy threads should be > 0 during load", - duringLoad.currentThreadsBusy, greaterThanOrEqualTo(1)); - assertThat("Busy threads <= current threads", - duringLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadCount)); - assertThat("Current threads <= max threads", - duringLoad.currentThreadCount, lessThanOrEqualTo(duringLoad.maxThreads)); + + assertThat("current threads during load", duringLoad.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); + assertThat("busy threads during load", duringLoad.currentThreadsBusy, greaterThanOrEqualTo(1)); + assertThat("busy threads during load", duringLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadCount)); + assertThat("current threads during load", duringLoad.currentThreadCount, lessThanOrEqualTo(duringLoad.maxThreads)); // Wait for completion CompletableFuture.allOf(futures).get(30, TimeUnit.SECONDS); @@ -110,33 +106,30 @@ void testThreadPoolMetricsUnderLoad() throws Exception { // Check metrics after load Thread.sleep(1000); // Let threads settle ThreadPoolMetrics afterLoad = getThreadPoolMetrics(); - - assertThat("Busy threads should decrease after load", - afterLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadsBusy)); - assertTrue(afterLoad.currentThreadsBusy >= 0); - assertTrue(afterLoad.currentThreadsBusy <= afterLoad.currentThreadCount); + + assertThat("busy threads after load", afterLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadsBusy)); + assertThat("busy threads after load", afterLoad.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("busy threads after load", afterLoad.currentThreadsBusy, lessThanOrEqualTo(afterLoad.currentThreadCount)); } @Test void testThreadPoolMetricsBaseline() throws Exception { ThreadPoolMetrics metrics = getThreadPoolMetrics(); - + // Basic sanity checks - assertTrue(metrics.currentThreadCount >= 0, "Current thread count should be non-negative"); - assertTrue(metrics.currentThreadsBusy >= 0, "Busy thread count should be non-negative"); - assertTrue(metrics.maxThreads > 0, "Max threads should be positive"); - + assertThat("current thread count", metrics.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("busy thread count", metrics.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("max threads", metrics.maxThreads, greaterThan(0)); + // Logical consistency - assertTrue(metrics.currentThreadsBusy <= metrics.currentThreadCount, - "Busy threads should not exceed current threads"); - assertTrue(metrics.currentThreadCount <= metrics.maxThreads, - "Current threads should not exceed max threads"); + assertThat("busy threads", metrics.currentThreadsBusy, lessThanOrEqualTo(metrics.currentThreadCount)); + assertThat("current threads", metrics.currentThreadCount, lessThanOrEqualTo(metrics.maxThreads)); } @Test void testThreadPoolMetricsWithSequentialRequests() throws Exception { ThreadPoolMetrics baseline = getThreadPoolMetrics(); - + // Make sequential requests to see if metrics respond for (int i = 0; i < 5; i++) { HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/test"); @@ -145,23 +138,23 @@ void testThreadPoolMetricsWithSequentialRequests() throws Exception { conn.disconnect(); Thread.sleep(100); } - + ThreadPoolMetrics afterSequential = getThreadPoolMetrics(); - + // Metrics should remain consistent - assertTrue(afterSequential.currentThreadCount >= 0); - assertTrue(afterSequential.currentThreadsBusy >= 0); - assertTrue(afterSequential.currentThreadsBusy <= afterSequential.currentThreadCount); + assertThat("current thread count after sequential", afterSequential.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("busy threads after sequential", afterSequential.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("busy threads after sequential", afterSequential.currentThreadsBusy, lessThanOrEqualTo(afterSequential.currentThreadCount)); } @Test void testThreadPoolMetricsWithBurstLoad() throws Exception { ThreadPoolMetrics baseline = getThreadPoolMetrics(); - + // Create burst of quick requests ExecutorService executor = Executors.newFixedThreadPool(50); CompletableFuture[] futures = new CompletableFuture[100]; - + for (int i = 0; i < futures.length; i++) { futures[i] = CompletableFuture.runAsync(() -> { try { @@ -177,53 +170,51 @@ void testThreadPoolMetricsWithBurstLoad() throws Exception { // Check metrics immediately during burst ThreadPoolMetrics duringBurst = getThreadPoolMetrics(); - + // Wait for completion CompletableFuture.allOf(futures).get(15, TimeUnit.SECONDS); executor.shutdown(); - + ThreadPoolMetrics afterBurst = getThreadPoolMetrics(); - + // Validate all metrics remain within bounds - assertTrue(duringBurst.currentThreadCount >= baseline.currentThreadCount); - assertTrue(duringBurst.currentThreadsBusy >= 0); - assertTrue(duringBurst.currentThreadsBusy <= duringBurst.currentThreadCount); - - assertTrue(afterBurst.currentThreadCount >= 0); - assertTrue(afterBurst.currentThreadsBusy >= 0); - assertTrue(afterBurst.currentThreadsBusy <= afterBurst.currentThreadCount); + assertThat("current threads during burst", duringBurst.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); + assertThat("busy threads during burst", duringBurst.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("busy threads during burst", duringBurst.currentThreadsBusy, lessThanOrEqualTo(duringBurst.currentThreadCount)); + + assertThat("current threads after burst", afterBurst.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("busy threads after burst", afterBurst.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("busy threads after burst", afterBurst.currentThreadsBusy, lessThanOrEqualTo(afterBurst.currentThreadCount)); } @Test void testThreadPoolMetricsConsistency() throws Exception { // Take multiple samples to check for consistency ThreadPoolMetrics[] samples = new ThreadPoolMetrics[5]; - + for (int i = 0; i < samples.length; i++) { samples[i] = getThreadPoolMetrics(); Thread.sleep(200); } - + // All samples should have consistent logical relationships for (ThreadPoolMetrics sample : samples) { - assertTrue(sample.currentThreadCount >= 0, "Current threads >= 0"); - assertTrue(sample.currentThreadsBusy >= 0, "Busy threads >= 0"); - assertTrue(sample.maxThreads > 0, "Max threads > 0"); - assertTrue(sample.currentThreadsBusy <= sample.currentThreadCount, - "Busy <= Current"); - assertTrue(sample.currentThreadCount <= sample.maxThreads, - "Current <= Max"); + assertThat("current threads", sample.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("busy threads", sample.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("max threads", sample.maxThreads, greaterThan(0)); + assertThat("busy threads", sample.currentThreadsBusy, lessThanOrEqualTo(sample.currentThreadCount)); + assertThat("current threads", sample.currentThreadCount, lessThanOrEqualTo(sample.maxThreads)); } } @Test void testThreadPoolMetricsUnderSustainedLoad() throws Exception { ThreadPoolMetrics baseline = getThreadPoolMetrics(); - + // Create sustained load for longer period ExecutorService executor = Executors.newFixedThreadPool(8); CompletableFuture[] futures = new CompletableFuture[16]; - + for (int i = 0; i < futures.length; i++) { futures[i] = CompletableFuture.runAsync(() -> { try { @@ -243,41 +234,37 @@ void testThreadPoolMetricsUnderSustainedLoad() throws Exception { // Sample metrics during sustained load Thread.sleep(500); ThreadPoolMetrics sample1 = getThreadPoolMetrics(); - + Thread.sleep(1000); ThreadPoolMetrics sample2 = getThreadPoolMetrics(); - + Thread.sleep(1000); ThreadPoolMetrics sample3 = getThreadPoolMetrics(); - + // Wait for completion CompletableFuture.allOf(futures).get(60, TimeUnit.SECONDS); executor.shutdown(); - + ThreadPoolMetrics afterSustained = getThreadPoolMetrics(); - + // During sustained load, we should see consistent thread usage ThreadPoolMetrics[] samples = {sample1, sample2, sample3}; for (int i = 0; i < samples.length; i++) { ThreadPoolMetrics sample = samples[i]; - assertTrue(sample.currentThreadCount >= baseline.currentThreadCount, - "Sample " + i + ": threads should increase under load"); - assertTrue(sample.currentThreadsBusy >= 0, - "Sample " + i + ": busy threads should be non-negative"); - assertTrue(sample.currentThreadsBusy <= sample.currentThreadCount, - "Sample " + i + ": busy <= current"); + assertThat("sample " + i + " current threads", sample.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); + assertThat("sample " + i + " busy threads", sample.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("sample " + i + " busy threads", sample.currentThreadsBusy, lessThanOrEqualTo(sample.currentThreadCount)); } - + // After load, busy threads should decrease - assertTrue(afterSustained.currentThreadsBusy <= sample3.currentThreadsBusy, - "Busy threads should decrease after sustained load"); + assertThat("busy threads after sustained load", afterSustained.currentThreadsBusy, lessThanOrEqualTo(sample3.currentThreadsBusy)); } private static File createDeployment() throws IOException { WebArchive war = ShrinkWrap.create(WebArchive.class, APP_NAME + ".war") .addClass(TestServlet.class) .addClass(SlowServlet.class); - + File warFile = new File(System.getProperty("java.io.tmpdir"), APP_NAME + ".war"); war.as(ZipExporter.class).exportTo(warFile, true); return warFile; @@ -286,18 +273,18 @@ private static File createDeployment() throws IOException { private ThreadPoolMetrics getThreadPoolMetrics() { return getThreadPoolMetrics("http-listener-1"); } - + private ThreadPoolMetrics getThreadPoolMetrics(String listenerName) { // First, let's see what monitoring data is actually available AsadminResult listResult = ASADMIN.exec("list", "*thread*"); System.out.println("Available thread monitoring paths: " + listResult.getStdOut()); - + // Try to get metrics from the specified listener String basePath = "server.network." + listenerName + ".thread-pool"; AsadminResult currentResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadcount"); AsadminResult busyResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadsbusy"); AsadminResult maxResult = ASADMIN.exec("get", "-m", basePath + ".maxthreads"); - + return new ThreadPoolMetrics( extractValue(currentResult.getStdOut(), "currentthreadcount"), extractValue(busyResult.getStdOut(), "currentthreadsbusy"), @@ -345,17 +332,17 @@ private static class ThreadPoolMetrics { void testThreadPoolSizeIncrease() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); int originalMaxThreads = initial.maxThreads; - + try { // Increase pool size significantly int newMaxThreads = originalMaxThreads + 100; ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + newMaxThreads); - + Thread.sleep(2000); - + ThreadPoolMetrics afterIncrease = getThreadPoolMetrics(); - assertEquals(newMaxThreads, afterIncrease.maxThreads, "Max threads should reflect new configuration"); - + assertThat("max threads after increase", afterIncrease.maxThreads, equalTo(newMaxThreads)); + // Generate load to test the increased pool ExecutorService executor = Executors.newFixedThreadPool(50); for (int i = 0; i < 50; i++) { @@ -369,21 +356,21 @@ void testThreadPoolSizeIncrease() throws Exception { } }); } - + Thread.sleep(500); // Let load start ThreadPoolMetrics underLoad = getThreadPoolMetrics(); - + // Critical assertions: monitoring should report valid values - assertTrue(underLoad.currentThreadCount >= 0, "Current thread count should never be negative"); - assertTrue(underLoad.currentThreadCount > initial.currentThreadCount, "Thread count should increase under load"); - assertTrue(underLoad.currentThreadCount <= newMaxThreads, "Current <= max"); - + assertThat("current thread count under load", underLoad.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("current thread count under load", underLoad.currentThreadCount, greaterThan(initial.currentThreadCount)); + assertThat("current thread count under load", underLoad.currentThreadCount, lessThanOrEqualTo(newMaxThreads)); + // Should have approximately 50 threads active for 50 concurrent requests - assertTrue(underLoad.currentThreadCount >= 30, "Should have many threads active for 50 concurrent requests"); - + assertThat("current thread count under load", underLoad.currentThreadCount, greaterThanOrEqualTo(30)); + executor.shutdown(); Thread.sleep(3000); // Wait for completion - + } finally { ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); } @@ -393,15 +380,15 @@ void testThreadPoolSizeIncrease() throws Exception { void testThreadPoolSizeDecrease() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); int originalMaxThreads = initial.maxThreads; - + try { // First increase pool size to 100 ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=100"); Thread.sleep(2000); - + ThreadPoolMetrics afterIncrease = getThreadPoolMetrics(); - assertEquals(100, afterIncrease.maxThreads, "Max threads should be 100"); - + assertThat("max threads after increase", afterIncrease.maxThreads, equalTo(100)); + // Generate significant load to utilize the large pool ExecutorService executor = Executors.newFixedThreadPool(80); for (int i = 0; i < 80; i++) { @@ -415,40 +402,40 @@ void testThreadPoolSizeDecrease() throws Exception { } }); } - + Thread.sleep(1000); // Let load build up ThreadPoolMetrics underHeavyLoad = getThreadPoolMetrics(); - + // Verify the pool scaled up to handle heavy load - assertTrue(underHeavyLoad.currentThreadCount > initial.currentThreadCount, "Thread count should increase under heavy load"); - assertTrue(underHeavyLoad.currentThreadCount <= 100, "Current threads should not exceed 100"); - assertTrue(underHeavyLoad.currentThreadCount >= 0, "Current thread count should never be negative"); - + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, greaterThan(initial.currentThreadCount)); + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, lessThanOrEqualTo(100)); + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, greaterThanOrEqualTo(0)); + // Should have approximately 80 threads active (or close to it) - assertTrue(underHeavyLoad.currentThreadCount >= 50, "Should have many threads active for 80 concurrent requests"); - + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, greaterThanOrEqualTo(50)); + // Now decrease pool size to 10 while under load ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=10"); Thread.sleep(1000); - + ThreadPoolMetrics afterDecrease = getThreadPoolMetrics(); - assertEquals(10, afterDecrease.maxThreads, "Max threads should be 10"); - + assertThat("max threads after decrease", afterDecrease.maxThreads, equalTo(10)); + // Critical assertions: thread count should remain valid - assertTrue(afterDecrease.currentThreadCount >= 0, "Current thread count should never be negative"); - assertTrue(afterDecrease.currentThreadsBusy >= 0, "Busy threads should be non-negative"); - + assertThat("current threads after decrease", afterDecrease.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("busy threads after decrease", afterDecrease.currentThreadsBusy, greaterThanOrEqualTo(0)); + // Since we had 80 concurrent requests, current threads should still be high // (threads don't disappear instantly when pool size is reduced) - assertTrue(afterDecrease.currentThreadCount >= 10, "Should still have many active threads from previous load"); - + assertThat("current threads after decrease", afterDecrease.currentThreadCount, greaterThanOrEqualTo(10)); + executor.shutdown(); Thread.sleep(4000); // Wait for requests to complete - + ThreadPoolMetrics afterCompletion = getThreadPoolMetrics(); - assertTrue(afterCompletion.currentThreadCount >= 0, "Thread count should never be negative"); - assertTrue(afterCompletion.currentThreadCount <= 10, "Eventually current threads should respect new max"); - + assertThat("current threads after completion", afterCompletion.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("current threads after completion", afterCompletion.currentThreadCount, lessThanOrEqualTo(10)); + } finally { ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); } @@ -458,22 +445,22 @@ void testThreadPoolSizeDecrease() throws Exception { void testThreadPoolSizeCycling() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); int originalMaxThreads = initial.maxThreads; - + try { int[] testSizes = {originalMaxThreads + 3, originalMaxThreads - 1, originalMaxThreads + 7, originalMaxThreads}; - + for (int testSize : testSizes) { testSize = Math.max(1, testSize); - + ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + testSize); Thread.sleep(1500); - + ThreadPoolMetrics metrics = getThreadPoolMetrics(); - assertEquals(testSize, metrics.maxThreads, "Max threads should match configured size: " + testSize); - assertTrue(metrics.currentThreadCount <= testSize, "Current threads should not exceed max: " + testSize); - assertTrue(metrics.currentThreadsBusy <= metrics.currentThreadCount, "Busy <= current for size: " + testSize); + assertThat("max threads", metrics.maxThreads, equalTo(testSize)); + assertThat("current threads", metrics.currentThreadCount, lessThanOrEqualTo(testSize)); + assertThat("busy threads", metrics.currentThreadsBusy, lessThanOrEqualTo(metrics.currentThreadCount)); } - + } finally { ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); } @@ -483,15 +470,15 @@ void testThreadPoolSizeCycling() throws Exception { void testAdminListenerThreadPoolMetrics() throws Exception { // Get baseline metrics for admin-listener ThreadPoolMetrics adminBaseline = getThreadPoolMetrics("admin-listener"); - + // Verify admin listener has valid baseline metrics - assertTrue(adminBaseline.currentThreadCount >= 0, "Admin listener should have valid thread count"); - assertTrue(adminBaseline.maxThreads > 0, "Admin listener should have positive max threads"); - assertTrue(adminBaseline.currentThreadsBusy >= 0, "Admin listener should have valid busy count"); - + assertThat("admin listener current threads", adminBaseline.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("admin listener max threads", adminBaseline.maxThreads, greaterThan(0)); + assertThat("admin listener busy threads", adminBaseline.currentThreadsBusy, greaterThanOrEqualTo(0)); + // Create load on admin port (4848) using asadmin commands ExecutorService executor = Executors.newFixedThreadPool(3); - + // Submit multiple concurrent admin requests for (int i = 0; i < 5; i++) { executor.submit(() -> { @@ -504,85 +491,82 @@ void testAdminListenerThreadPoolMetrics() throws Exception { } }); } - + Thread.sleep(500); // Let requests start - + // Check metrics during admin load ThreadPoolMetrics adminUnderLoad = getThreadPoolMetrics("admin-listener"); - + // Admin listener should maintain valid metrics under load - assertTrue(adminUnderLoad.currentThreadCount >= 0, "Admin listener should maintain valid thread count under load"); - assertTrue(adminUnderLoad.currentThreadsBusy <= adminUnderLoad.currentThreadCount, - "Admin listener: Busy threads should not exceed current threads"); - assertTrue(adminUnderLoad.maxThreads > 0, "Admin listener should maintain positive max threads"); - + assertThat("admin listener current threads under load", adminUnderLoad.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("admin listener busy threads under load", adminUnderLoad.currentThreadsBusy, lessThanOrEqualTo(adminUnderLoad.currentThreadCount)); + assertThat("admin listener max threads under load", adminUnderLoad.maxThreads, greaterThan(0)); + executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - + Thread.sleep(1000); // Let threads settle - + // Final metrics check ThreadPoolMetrics adminFinal = getThreadPoolMetrics("admin-listener"); - assertTrue(adminFinal.currentThreadCount >= 0, "Admin listener final thread count should be valid"); - assertTrue(adminFinal.currentThreadsBusy >= 0, "Admin listener final busy count should be valid"); + assertThat("admin listener final current threads", adminFinal.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("admin listener final busy threads", adminFinal.currentThreadsBusy, greaterThanOrEqualTo(0)); } @Test void testDualListenerThreadPoolMetrics() throws Exception { // First create http-listener-2 if it doesn't exist - AsadminResult createResult = ASADMIN.exec("create-http-listener", - "--listenerport=8081", "--listeneraddress=0.0.0.0", + AsadminResult createResult = ASADMIN.exec("create-http-listener", + "--listenerport=8081", "--listeneraddress=0.0.0.0", "--defaultvs=server", "http-listener-2"); - + try { Thread.sleep(2000); // Allow listener to initialize - + // Get baseline metrics for both listeners ThreadPoolMetrics listener1Baseline = getThreadPoolMetrics("http-listener-1"); ThreadPoolMetrics listener2Baseline = getThreadPoolMetrics("http-listener-2"); - + // Verify both listeners have valid baseline metrics - assertTrue(listener1Baseline.currentThreadCount >= 0, "Listener 1 should have valid thread count"); - assertTrue(listener2Baseline.currentThreadCount >= 0, "Listener 2 should have valid thread count"); - + assertThat("listener 1 current threads", listener1Baseline.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("listener 2 current threads", listener2Baseline.currentThreadCount, greaterThanOrEqualTo(0)); + // Create load on both listeners simultaneously ExecutorService executor = Executors.newFixedThreadPool(10); - + // Submit requests to both ports for (int i = 0; i < 5; i++) { executor.submit(() -> makeRequest("http://localhost:4848/" + APP_NAME + "/hello")); executor.submit(() -> makeRequest("http://localhost:8081/" + APP_NAME + "/hello")); } - + Thread.sleep(500); // Let requests start - + // Check metrics during dual load ThreadPoolMetrics listener1UnderLoad = getThreadPoolMetrics("http-listener-1"); ThreadPoolMetrics listener2UnderLoad = getThreadPoolMetrics("http-listener-2"); - + // Both listeners should show activity - assertTrue(listener1UnderLoad.currentThreadCount >= 0, "Listener 1 should maintain valid thread count under load"); - assertTrue(listener2UnderLoad.currentThreadCount >= 0, "Listener 2 should maintain valid thread count under load"); - + assertThat("listener 1 current threads under load", listener1UnderLoad.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("listener 2 current threads under load", listener2UnderLoad.currentThreadCount, greaterThanOrEqualTo(0)); + // Thread counts should be consistent with busy counts - assertTrue(listener1UnderLoad.currentThreadsBusy <= listener1UnderLoad.currentThreadCount, - "Listener 1: Busy threads should not exceed current threads"); - assertTrue(listener2UnderLoad.currentThreadsBusy <= listener2UnderLoad.currentThreadCount, - "Listener 2: Busy threads should not exceed current threads"); - + assertThat("listener 1 busy threads under load", listener1UnderLoad.currentThreadsBusy, lessThanOrEqualTo(listener1UnderLoad.currentThreadCount)); + assertThat("listener 2 busy threads under load", listener2UnderLoad.currentThreadsBusy, lessThanOrEqualTo(listener2UnderLoad.currentThreadCount)); + executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - + Thread.sleep(1000); // Let threads settle - + // Final metrics check ThreadPoolMetrics listener1Final = getThreadPoolMetrics("http-listener-1"); ThreadPoolMetrics listener2Final = getThreadPoolMetrics("http-listener-2"); - + // Both should have valid final states - assertTrue(listener1Final.currentThreadCount >= 0, "Listener 1 final thread count should be valid"); - assertTrue(listener2Final.currentThreadCount >= 0, "Listener 2 final thread count should be valid"); - + assertThat("listener 1 final current threads", listener1Final.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("listener 2 final current threads", listener2Final.currentThreadCount, greaterThanOrEqualTo(0)); + } finally { // Clean up http-listener-2 ASADMIN.exec("delete-http-listener", "http-listener-2"); @@ -593,13 +577,13 @@ void testDualListenerThreadPoolMetrics() throws Exception { void testThreadPoolSizeUnderLoad() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); int originalMaxThreads = initial.maxThreads; - + try { // Start with increased pool size int largePoolSize = originalMaxThreads + 5; ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + largePoolSize); Thread.sleep(2000); - + // Generate load ExecutorService executor = Executors.newFixedThreadPool(8); for (int i = 0; i < 8; i++) { @@ -613,21 +597,21 @@ void testThreadPoolSizeUnderLoad() throws Exception { } }); } - + Thread.sleep(500); ThreadPoolMetrics duringLoad = getThreadPoolMetrics(); - + // Decrease pool size while under load int smallPoolSize = Math.max(3, originalMaxThreads - 1); ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + smallPoolSize); Thread.sleep(1000); - + ThreadPoolMetrics afterResize = getThreadPoolMetrics(); - assertEquals(smallPoolSize, afterResize.maxThreads, "Max threads should reflect new size even under load"); - + assertThat("max threads after resize", afterResize.maxThreads, equalTo(smallPoolSize)); + executor.shutdown(); Thread.sleep(4000); - + } finally { ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); } From fc3b3e53a54304be9dacc7d2b9d720daab9cfc4d Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Sun, 26 Oct 2025 19:57:39 +0100 Subject: [PATCH 13/18] Fix the testThreadPoolSizeCycling test --- .../main/test/app/monitoring/SlowServlet.java | 10 +- .../main/test/app/monitoring/TestServlet.java | 8 +- .../monitoring/ThreadPoolMonitoringTest.java | 207 ++++++++++-------- 3 files changed, 127 insertions(+), 98 deletions(-) diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java index 02bf5b2206d..c2bf4a67f36 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java @@ -10,27 +10,27 @@ package org.glassfish.main.test.app.monitoring; -import java.io.IOException; -import java.io.PrintWriter; - import jakarta.servlet.ServletException; import jakarta.servlet.annotation.WebServlet; import jakarta.servlet.http.HttpServlet; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; + @WebServlet("/slow") public class SlowServlet extends HttpServlet { @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { try { Thread.sleep(2000); // 2 second delay to keep threads busy } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - + resp.setContentType("text/plain"); try (PrintWriter writer = resp.getWriter()) { writer.println("Slow response completed"); diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java index cbd133dce6b..6e52ed6b50b 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java @@ -10,20 +10,20 @@ package org.glassfish.main.test.app.monitoring; -import java.io.IOException; -import java.io.PrintWriter; - import jakarta.servlet.ServletException; import jakarta.servlet.annotation.WebServlet; import jakarta.servlet.http.HttpServlet; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.PrintWriter; + @WebServlet("/test") public class TestServlet extends HttpServlet { @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { resp.setContentType("text/plain"); try (PrintWriter writer = resp.getWriter()) { diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java index 40620dcf623..81c2cdbf7a9 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java @@ -18,6 +18,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; import org.glassfish.main.itest.tools.GlassFishTestEnvironment; import org.glassfish.main.itest.tools.asadmin.Asadmin; @@ -38,6 +39,7 @@ public class ThreadPoolMonitoringTest { + private static final Logger LOG = Logger.getLogger(ThreadPoolMonitoringTest.class.getName()); private static final String APP_NAME = "threadpool-test"; private static final Asadmin ASADMIN = GlassFishTestEnvironment.getAsadmin(); @@ -45,7 +47,7 @@ public class ThreadPoolMonitoringTest { static void deployApp() throws IOException { File warFile = createDeployment(); try { - AsadminResult result = ASADMIN.exec("deploy", warFile.getAbsolutePath()); + AsadminResult result = ASADMIN.exec("deploy", "--force", warFile.getAbsolutePath()); assertThat(result, AsadminResultMatcher.asadminOK()); // Enable monitoring @@ -69,9 +71,9 @@ static void undeployApp() { void testThreadPoolMetricsUnderLoad() throws Exception { // Get baseline metrics ThreadPoolMetrics baseline = getThreadPoolMetrics(); - assertThat("baseline current thread count", baseline.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("baseline busy threads", baseline.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("baseline busy threads", baseline.currentThreadsBusy, lessThanOrEqualTo(baseline.currentThreadCount)); + assertThat("baseline current thread count", baseline.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("baseline busy threads", baseline.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("baseline busy threads", baseline.currentThreadsBusy(), lessThanOrEqualTo(baseline.currentThreadCount())); // Generate concurrent load ExecutorService executor = Executors.newFixedThreadPool(10); @@ -94,10 +96,10 @@ void testThreadPoolMetricsUnderLoad() throws Exception { Thread.sleep(100); // Let requests start ThreadPoolMetrics duringLoad = getThreadPoolMetrics(); - assertThat("current threads during load", duringLoad.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); - assertThat("busy threads during load", duringLoad.currentThreadsBusy, greaterThanOrEqualTo(1)); - assertThat("busy threads during load", duringLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadCount)); - assertThat("current threads during load", duringLoad.currentThreadCount, lessThanOrEqualTo(duringLoad.maxThreads)); + assertThat("current threads during load", duringLoad.currentThreadCount(), greaterThanOrEqualTo(baseline.currentThreadCount())); + assertThat("busy threads during load", duringLoad.currentThreadsBusy(), greaterThanOrEqualTo(1)); + assertThat("busy threads during load", duringLoad.currentThreadsBusy(), lessThanOrEqualTo(duringLoad.currentThreadCount())); + assertThat("current threads during load", duringLoad.currentThreadCount(), lessThanOrEqualTo(duringLoad.maxThreads())); // Wait for completion CompletableFuture.allOf(futures).get(30, TimeUnit.SECONDS); @@ -107,9 +109,9 @@ void testThreadPoolMetricsUnderLoad() throws Exception { Thread.sleep(1000); // Let threads settle ThreadPoolMetrics afterLoad = getThreadPoolMetrics(); - assertThat("busy threads after load", afterLoad.currentThreadsBusy, lessThanOrEqualTo(duringLoad.currentThreadsBusy)); - assertThat("busy threads after load", afterLoad.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("busy threads after load", afterLoad.currentThreadsBusy, lessThanOrEqualTo(afterLoad.currentThreadCount)); + assertThat("busy threads after load", afterLoad.currentThreadsBusy(), lessThanOrEqualTo(duringLoad.currentThreadsBusy())); + assertThat("busy threads after load", afterLoad.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("busy threads after load", afterLoad.currentThreadsBusy(), lessThanOrEqualTo(afterLoad.currentThreadCount())); } @Test @@ -117,13 +119,13 @@ void testThreadPoolMetricsBaseline() throws Exception { ThreadPoolMetrics metrics = getThreadPoolMetrics(); // Basic sanity checks - assertThat("current thread count", metrics.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("busy thread count", metrics.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("max threads", metrics.maxThreads, greaterThan(0)); + assertThat("current thread count", metrics.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("busy thread count", metrics.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("max threads", metrics.maxThreads(), greaterThan(0)); // Logical consistency - assertThat("busy threads", metrics.currentThreadsBusy, lessThanOrEqualTo(metrics.currentThreadCount)); - assertThat("current threads", metrics.currentThreadCount, lessThanOrEqualTo(metrics.maxThreads)); + assertThat("busy threads", metrics.currentThreadsBusy(), lessThanOrEqualTo(metrics.currentThreadCount())); + assertThat("current threads", metrics.currentThreadCount(), lessThanOrEqualTo(metrics.maxThreads())); } @Test @@ -142,9 +144,9 @@ void testThreadPoolMetricsWithSequentialRequests() throws Exception { ThreadPoolMetrics afterSequential = getThreadPoolMetrics(); // Metrics should remain consistent - assertThat("current thread count after sequential", afterSequential.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("busy threads after sequential", afterSequential.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("busy threads after sequential", afterSequential.currentThreadsBusy, lessThanOrEqualTo(afterSequential.currentThreadCount)); + assertThat("current thread count after sequential", afterSequential.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("busy threads after sequential", afterSequential.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("busy threads after sequential", afterSequential.currentThreadsBusy(), lessThanOrEqualTo(afterSequential.currentThreadCount())); } @Test @@ -178,13 +180,13 @@ void testThreadPoolMetricsWithBurstLoad() throws Exception { ThreadPoolMetrics afterBurst = getThreadPoolMetrics(); // Validate all metrics remain within bounds - assertThat("current threads during burst", duringBurst.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); - assertThat("busy threads during burst", duringBurst.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("busy threads during burst", duringBurst.currentThreadsBusy, lessThanOrEqualTo(duringBurst.currentThreadCount)); + assertThat("current threads during burst", duringBurst.currentThreadCount(), greaterThanOrEqualTo(baseline.currentThreadCount())); + assertThat("busy threads during burst", duringBurst.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("busy threads during burst", duringBurst.currentThreadsBusy(), lessThanOrEqualTo(duringBurst.currentThreadCount())); - assertThat("current threads after burst", afterBurst.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("busy threads after burst", afterBurst.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("busy threads after burst", afterBurst.currentThreadsBusy, lessThanOrEqualTo(afterBurst.currentThreadCount)); + assertThat("current threads after burst", afterBurst.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("busy threads after burst", afterBurst.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("busy threads after burst", afterBurst.currentThreadsBusy(), lessThanOrEqualTo(afterBurst.currentThreadCount())); } @Test @@ -199,11 +201,11 @@ void testThreadPoolMetricsConsistency() throws Exception { // All samples should have consistent logical relationships for (ThreadPoolMetrics sample : samples) { - assertThat("current threads", sample.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("busy threads", sample.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("max threads", sample.maxThreads, greaterThan(0)); - assertThat("busy threads", sample.currentThreadsBusy, lessThanOrEqualTo(sample.currentThreadCount)); - assertThat("current threads", sample.currentThreadCount, lessThanOrEqualTo(sample.maxThreads)); + assertThat("current threads", sample.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("busy threads", sample.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("max threads", sample.maxThreads(), greaterThan(0)); + assertThat("busy threads", sample.currentThreadsBusy(), lessThanOrEqualTo(sample.currentThreadCount())); + assertThat("current threads", sample.currentThreadCount(), lessThanOrEqualTo(sample.maxThreads())); } } @@ -251,13 +253,13 @@ void testThreadPoolMetricsUnderSustainedLoad() throws Exception { ThreadPoolMetrics[] samples = {sample1, sample2, sample3}; for (int i = 0; i < samples.length; i++) { ThreadPoolMetrics sample = samples[i]; - assertThat("sample " + i + " current threads", sample.currentThreadCount, greaterThanOrEqualTo(baseline.currentThreadCount)); - assertThat("sample " + i + " busy threads", sample.currentThreadsBusy, greaterThanOrEqualTo(0)); - assertThat("sample " + i + " busy threads", sample.currentThreadsBusy, lessThanOrEqualTo(sample.currentThreadCount)); + assertThat("sample " + i + " current threads", sample.currentThreadCount(), greaterThanOrEqualTo(baseline.currentThreadCount())); + assertThat("sample " + i + " busy threads", sample.currentThreadsBusy(), greaterThanOrEqualTo(0)); + assertThat("sample " + i + " busy threads", sample.currentThreadsBusy(), lessThanOrEqualTo(sample.currentThreadCount())); } // After load, busy threads should decrease - assertThat("busy threads after sustained load", afterSustained.currentThreadsBusy, lessThanOrEqualTo(sample3.currentThreadsBusy)); + assertThat("busy threads after sustained load", afterSustained.currentThreadsBusy(), lessThanOrEqualTo(sample3.currentThreadsBusy())); } private static File createDeployment() throws IOException { @@ -284,22 +286,49 @@ private ThreadPoolMetrics getThreadPoolMetrics(String listenerName) { AsadminResult currentResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadcount"); AsadminResult busyResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadsbusy"); AsadminResult maxResult = ASADMIN.exec("get", "-m", basePath + ".maxthreads"); + AsadminResult minResult = ASADMIN.exec("get", "-m", basePath + ".minthreads"); return new ThreadPoolMetrics( - extractValue(currentResult.getStdOut(), "currentthreadcount"), - extractValue(busyResult.getStdOut(), "currentthreadsbusy"), - extractValue(maxResult.getStdOut(), "maxthreads") + extractMetric(currentResult.getStdOut(), "currentthreadcount"), + extractMetric(busyResult.getStdOut(), "currentthreadsbusy"), + extractMetric(minResult.getStdOut(), "minthreads"), + extractMetric(maxResult.getStdOut(), "maxthreads") ); } - private int extractValue(String output, String metric) { + private int extractMetric(String output, String metric) { String[] lines = output.split("\n"); for (String line : lines) { if (line.contains(metric + "-count")) { return Integer.parseInt(line.split("=")[1].trim()); } } - return 0; + return -1000; + } + + private ThreadPoolConfig getThreadPoolConfig() { + return getThreadPoolConfig("http-thread-pool"); + } + + private ThreadPoolConfig getThreadPoolConfig(String threadPoolName) { + String basePath = "configs.config.server-config.thread-pools.thread-pool.." + threadPoolName; + AsadminResult minResult = ASADMIN.exec("get", basePath + ".min-thread-pool-size"); + AsadminResult maxResult = ASADMIN.exec("get", basePath + ".max-thread-pool-size"); + + return new ThreadPoolConfig( + extractConfig(minResult.getStdOut(), "min-thread-pool-size"), + extractConfig(maxResult.getStdOut(), "max-thread-pool-size") + ); + } + + private int extractConfig(String output, String key) { + String[] lines = output.split("\n"); + for (String line : lines) { + if (line.contains(key)) { + return Integer.parseInt(line.split("=")[1].trim()); + } + } + return -1000; } private void makeRequest(String urlString) { @@ -316,22 +345,16 @@ private void makeRequest(String urlString) { } } - private static class ThreadPoolMetrics { - final int currentThreadCount; - final int currentThreadsBusy; - final int maxThreads; + private record ThreadPoolMetrics(int currentThreadCount, int currentThreadsBusy, int minThreads, int maxThreads) { + } - ThreadPoolMetrics(int currentThreadCount, int currentThreadsBusy, int maxThreads) { - this.currentThreadCount = currentThreadCount; - this.currentThreadsBusy = currentThreadsBusy; - this.maxThreads = maxThreads; - } + private record ThreadPoolConfig(int minThreads, int maxThreads) { } @Test void testThreadPoolSizeIncrease() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); - int originalMaxThreads = initial.maxThreads; + int originalMaxThreads = initial.maxThreads(); try { // Increase pool size significantly @@ -341,7 +364,7 @@ void testThreadPoolSizeIncrease() throws Exception { Thread.sleep(2000); ThreadPoolMetrics afterIncrease = getThreadPoolMetrics(); - assertThat("max threads after increase", afterIncrease.maxThreads, equalTo(newMaxThreads)); + assertThat("max threads after increase", afterIncrease.maxThreads(), equalTo(newMaxThreads)); // Generate load to test the increased pool ExecutorService executor = Executors.newFixedThreadPool(50); @@ -361,12 +384,12 @@ void testThreadPoolSizeIncrease() throws Exception { ThreadPoolMetrics underLoad = getThreadPoolMetrics(); // Critical assertions: monitoring should report valid values - assertThat("current thread count under load", underLoad.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("current thread count under load", underLoad.currentThreadCount, greaterThan(initial.currentThreadCount)); - assertThat("current thread count under load", underLoad.currentThreadCount, lessThanOrEqualTo(newMaxThreads)); + assertThat("current thread count under load", underLoad.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("current thread count under load", underLoad.currentThreadCount(), greaterThan(initial.currentThreadCount())); + assertThat("current thread count under load", underLoad.currentThreadCount(), lessThanOrEqualTo(newMaxThreads)); // Should have approximately 50 threads active for 50 concurrent requests - assertThat("current thread count under load", underLoad.currentThreadCount, greaterThanOrEqualTo(30)); + assertThat("current thread count under load", underLoad.currentThreadCount(), greaterThanOrEqualTo(30)); executor.shutdown(); Thread.sleep(3000); // Wait for completion @@ -379,7 +402,7 @@ void testThreadPoolSizeIncrease() throws Exception { @Test void testThreadPoolSizeDecrease() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); - int originalMaxThreads = initial.maxThreads; + int originalMaxThreads = initial.maxThreads(); try { // First increase pool size to 100 @@ -387,7 +410,7 @@ void testThreadPoolSizeDecrease() throws Exception { Thread.sleep(2000); ThreadPoolMetrics afterIncrease = getThreadPoolMetrics(); - assertThat("max threads after increase", afterIncrease.maxThreads, equalTo(100)); + assertThat("max threads after increase", afterIncrease.maxThreads(), equalTo(100)); // Generate significant load to utilize the large pool ExecutorService executor = Executors.newFixedThreadPool(80); @@ -407,34 +430,34 @@ void testThreadPoolSizeDecrease() throws Exception { ThreadPoolMetrics underHeavyLoad = getThreadPoolMetrics(); // Verify the pool scaled up to handle heavy load - assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, greaterThan(initial.currentThreadCount)); - assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, lessThanOrEqualTo(100)); - assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount(), greaterThan(initial.currentThreadCount())); + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount(), lessThanOrEqualTo(100)); + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount(), greaterThanOrEqualTo(0)); // Should have approximately 80 threads active (or close to it) - assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount, greaterThanOrEqualTo(50)); + assertThat("current threads under heavy load", underHeavyLoad.currentThreadCount(), greaterThanOrEqualTo(50)); // Now decrease pool size to 10 while under load ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=10"); Thread.sleep(1000); ThreadPoolMetrics afterDecrease = getThreadPoolMetrics(); - assertThat("max threads after decrease", afterDecrease.maxThreads, equalTo(10)); + assertThat("max threads after decrease", afterDecrease.maxThreads(), equalTo(10)); // Critical assertions: thread count should remain valid - assertThat("current threads after decrease", afterDecrease.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("busy threads after decrease", afterDecrease.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("current threads after decrease", afterDecrease.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("busy threads after decrease", afterDecrease.currentThreadsBusy(), greaterThanOrEqualTo(0)); // Since we had 80 concurrent requests, current threads should still be high // (threads don't disappear instantly when pool size is reduced) - assertThat("current threads after decrease", afterDecrease.currentThreadCount, greaterThanOrEqualTo(10)); + assertThat("current threads after decrease", afterDecrease.currentThreadCount(), greaterThanOrEqualTo(10)); executor.shutdown(); Thread.sleep(4000); // Wait for requests to complete ThreadPoolMetrics afterCompletion = getThreadPoolMetrics(); - assertThat("current threads after completion", afterCompletion.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("current threads after completion", afterCompletion.currentThreadCount, lessThanOrEqualTo(10)); + assertThat("current threads after completion", afterCompletion.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("current threads after completion", afterCompletion.currentThreadCount(), lessThanOrEqualTo(10)); } finally { ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); @@ -443,22 +466,28 @@ void testThreadPoolSizeDecrease() throws Exception { @Test void testThreadPoolSizeCycling() throws Exception { - ThreadPoolMetrics initial = getThreadPoolMetrics(); - int originalMaxThreads = initial.maxThreads; + ThreadPoolConfig initial = getThreadPoolConfig(); + int originalMinThreads = initial.minThreads(); + int originalMaxThreads = initial.maxThreads(); try { int[] testSizes = {originalMaxThreads + 3, originalMaxThreads - 1, originalMaxThreads + 7, originalMaxThreads}; + int i = 0; for (int testSize : testSizes) { + i++; + LOG.info("Case " + i + ": testSize=" + testSize); + testSize = Math.max(1, testSize); ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + testSize); Thread.sleep(1500); ThreadPoolMetrics metrics = getThreadPoolMetrics(); - assertThat("max threads", metrics.maxThreads, equalTo(testSize)); - assertThat("current threads", metrics.currentThreadCount, lessThanOrEqualTo(testSize)); - assertThat("busy threads", metrics.currentThreadsBusy, lessThanOrEqualTo(metrics.currentThreadCount)); + assertThat("max threads (testSize=" + testSize + ")", metrics.maxThreads(), equalTo(testSize)); + int currentThreadsMaxSize = Math.max(testSize, originalMinThreads); + assertThat("current threads (testSize=" + testSize + ")", metrics.currentThreadCount(), lessThanOrEqualTo(currentThreadsMaxSize)); + assertThat("busy threads (testSize=" + testSize + ")", metrics.currentThreadsBusy(), lessThanOrEqualTo(metrics.currentThreadCount())); } } finally { @@ -472,9 +501,9 @@ void testAdminListenerThreadPoolMetrics() throws Exception { ThreadPoolMetrics adminBaseline = getThreadPoolMetrics("admin-listener"); // Verify admin listener has valid baseline metrics - assertThat("admin listener current threads", adminBaseline.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("admin listener max threads", adminBaseline.maxThreads, greaterThan(0)); - assertThat("admin listener busy threads", adminBaseline.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("admin listener current threads", adminBaseline.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("admin listener max threads", adminBaseline.maxThreads(), greaterThan(0)); + assertThat("admin listener busy threads", adminBaseline.currentThreadsBusy(), greaterThanOrEqualTo(0)); // Create load on admin port (4848) using asadmin commands ExecutorService executor = Executors.newFixedThreadPool(3); @@ -498,9 +527,9 @@ void testAdminListenerThreadPoolMetrics() throws Exception { ThreadPoolMetrics adminUnderLoad = getThreadPoolMetrics("admin-listener"); // Admin listener should maintain valid metrics under load - assertThat("admin listener current threads under load", adminUnderLoad.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("admin listener busy threads under load", adminUnderLoad.currentThreadsBusy, lessThanOrEqualTo(adminUnderLoad.currentThreadCount)); - assertThat("admin listener max threads under load", adminUnderLoad.maxThreads, greaterThan(0)); + assertThat("admin listener current threads under load", adminUnderLoad.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("admin listener busy threads under load", adminUnderLoad.currentThreadsBusy(), lessThanOrEqualTo(adminUnderLoad.currentThreadCount())); + assertThat("admin listener max threads under load", adminUnderLoad.maxThreads(), greaterThan(0)); executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); @@ -509,8 +538,8 @@ void testAdminListenerThreadPoolMetrics() throws Exception { // Final metrics check ThreadPoolMetrics adminFinal = getThreadPoolMetrics("admin-listener"); - assertThat("admin listener final current threads", adminFinal.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("admin listener final busy threads", adminFinal.currentThreadsBusy, greaterThanOrEqualTo(0)); + assertThat("admin listener final current threads", adminFinal.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("admin listener final busy threads", adminFinal.currentThreadsBusy(), greaterThanOrEqualTo(0)); } @Test @@ -528,8 +557,8 @@ void testDualListenerThreadPoolMetrics() throws Exception { ThreadPoolMetrics listener2Baseline = getThreadPoolMetrics("http-listener-2"); // Verify both listeners have valid baseline metrics - assertThat("listener 1 current threads", listener1Baseline.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("listener 2 current threads", listener2Baseline.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("listener 1 current threads", listener1Baseline.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("listener 2 current threads", listener2Baseline.currentThreadCount(), greaterThanOrEqualTo(0)); // Create load on both listeners simultaneously ExecutorService executor = Executors.newFixedThreadPool(10); @@ -547,12 +576,12 @@ void testDualListenerThreadPoolMetrics() throws Exception { ThreadPoolMetrics listener2UnderLoad = getThreadPoolMetrics("http-listener-2"); // Both listeners should show activity - assertThat("listener 1 current threads under load", listener1UnderLoad.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("listener 2 current threads under load", listener2UnderLoad.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("listener 1 current threads under load", listener1UnderLoad.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("listener 2 current threads under load", listener2UnderLoad.currentThreadCount(), greaterThanOrEqualTo(0)); // Thread counts should be consistent with busy counts - assertThat("listener 1 busy threads under load", listener1UnderLoad.currentThreadsBusy, lessThanOrEqualTo(listener1UnderLoad.currentThreadCount)); - assertThat("listener 2 busy threads under load", listener2UnderLoad.currentThreadsBusy, lessThanOrEqualTo(listener2UnderLoad.currentThreadCount)); + assertThat("listener 1 busy threads under load", listener1UnderLoad.currentThreadsBusy(), lessThanOrEqualTo(listener1UnderLoad.currentThreadCount())); + assertThat("listener 2 busy threads under load", listener2UnderLoad.currentThreadsBusy(), lessThanOrEqualTo(listener2UnderLoad.currentThreadCount())); executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); @@ -564,8 +593,8 @@ void testDualListenerThreadPoolMetrics() throws Exception { ThreadPoolMetrics listener2Final = getThreadPoolMetrics("http-listener-2"); // Both should have valid final states - assertThat("listener 1 final current threads", listener1Final.currentThreadCount, greaterThanOrEqualTo(0)); - assertThat("listener 2 final current threads", listener2Final.currentThreadCount, greaterThanOrEqualTo(0)); + assertThat("listener 1 final current threads", listener1Final.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("listener 2 final current threads", listener2Final.currentThreadCount(), greaterThanOrEqualTo(0)); } finally { // Clean up http-listener-2 @@ -576,7 +605,7 @@ void testDualListenerThreadPoolMetrics() throws Exception { @Test void testThreadPoolSizeUnderLoad() throws Exception { ThreadPoolMetrics initial = getThreadPoolMetrics(); - int originalMaxThreads = initial.maxThreads; + int originalMaxThreads = initial.maxThreads(); try { // Start with increased pool size @@ -607,7 +636,7 @@ void testThreadPoolSizeUnderLoad() throws Exception { Thread.sleep(1000); ThreadPoolMetrics afterResize = getThreadPoolMetrics(); - assertThat("max threads after resize", afterResize.maxThreads, equalTo(smallPoolSize)); + assertThat("max threads after resize", afterResize.maxThreads(), equalTo(smallPoolSize)); executor.shutdown(); Thread.sleep(4000); From ff162d50cf10e28119e1b0595afef59cee601454 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Mon, 27 Oct 2025 22:45:02 +0100 Subject: [PATCH 14/18] Fixes in thread pool monitoring - do not unregister probes when deleting listener, unless it's the last listener - count busy threads in the monitor and pass info to stats (maybe not necessary) - catch out of bounds exception in probes, which sometimes happens, probably because of a probe is removed concurrently from the array while iterating through the collection - global stats updates with the setCurrentThreadCountEvent event --- .../main/test/app/monitoring/SlowServlet.java | 4 +- .../main/test/app/monitoring/TestServlet.java | 0 .../monitoring/ThreadPoolMonitoringTest.java | 129 +++++++++++++----- .../impl/GlassfishNetworkListener.java | 13 +- .../v3/services/impl/GrizzlyProxy.java | 8 ++ .../v3/services/impl/GrizzlyService.java | 3 +- .../v3/services/impl/NetworkProxy.java | 8 ++ .../impl/monitor/ThreadPoolMonitor.java | 15 +- .../probes/ThreadPoolProbeProvider.java | 6 +- .../stats/ThreadPoolStatsProvider.java | 11 +- .../stats/ThreadPoolStatsProviderGlobal.java | 18 ++- .../flashlight/provider/FlashlightProbe.java | 14 +- .../grizzly/config/GrizzlyListener.java | 4 + 13 files changed, 175 insertions(+), 58 deletions(-) rename appserver/tests/application/src/{test => main}/java/org/glassfish/main/test/app/monitoring/SlowServlet.java (86%) rename appserver/tests/application/src/{test => main}/java/org/glassfish/main/test/app/monitoring/TestServlet.java (100%) diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java b/appserver/tests/application/src/main/java/org/glassfish/main/test/app/monitoring/SlowServlet.java similarity index 86% rename from appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java rename to appserver/tests/application/src/main/java/org/glassfish/main/test/app/monitoring/SlowServlet.java index c2bf4a67f36..18f323176f6 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/SlowServlet.java +++ b/appserver/tests/application/src/main/java/org/glassfish/main/test/app/monitoring/SlowServlet.java @@ -25,8 +25,10 @@ public class SlowServlet extends HttpServlet { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String delayParam = req.getParameter("delay"); + int delay = delayParam != null ? Integer.parseInt(delayParam) : 5000; try { - Thread.sleep(2000); // 2 second delay to keep threads busy + Thread.sleep(delay); // delay to keep threads busy } catch (InterruptedException e) { Thread.currentThread().interrupt(); } diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java b/appserver/tests/application/src/main/java/org/glassfish/main/test/app/monitoring/TestServlet.java similarity index 100% rename from appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/TestServlet.java rename to appserver/tests/application/src/main/java/org/glassfish/main/test/app/monitoring/TestServlet.java diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java index 81c2cdbf7a9..5352349fc1d 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java @@ -7,17 +7,18 @@ * * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 */ - package org.glassfish.main.test.app.monitoring; import java.io.File; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import org.glassfish.main.itest.tools.GlassFishTestEnvironment; @@ -28,15 +29,24 @@ import org.jboss.shrinkwrap.api.exporter.ZipExporter; import org.jboss.shrinkwrap.api.spec.WebArchive; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.AfterTestExecutionCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.RegisterExtension; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.junit.jupiter.api.Assumptions.assumeFalse; +@TestMethodOrder(MethodOrderer.MethodName.class) public class ThreadPoolMonitoringTest { private static final Logger LOG = Logger.getLogger(ThreadPoolMonitoringTest.class.getName()); @@ -69,6 +79,13 @@ static void undeployApp() { @Test void testThreadPoolMetricsUnderLoad() throws Exception { + + AsadminResult createResult = ASADMIN.exec("create-http-listener", + "--listenerport=8081", "--listeneraddress=0.0.0.0", + "--defaultvs=server", "http-listener-test"); +// ASADMIN.exec("delete-http-listener", "http-listener-test"); + + // Get baseline metrics ThreadPoolMetrics baseline = getThreadPoolMetrics(); assertThat("baseline current thread count", baseline.currentThreadCount(), greaterThanOrEqualTo(0)); @@ -79,12 +96,16 @@ void testThreadPoolMetricsUnderLoad() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(10); CompletableFuture[] futures = new CompletableFuture[20]; + AtomicInteger activeRequests = new AtomicInteger(0); + for (int i = 0; i < futures.length; i++) { futures[i] = CompletableFuture.runAsync(() -> { try { HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/slow"); conn.setRequestMethod("GET"); + activeRequests.incrementAndGet(); conn.getResponseCode(); + activeRequests.decrementAndGet(); conn.disconnect(); } catch (Exception e) { throw new RuntimeException(e); @@ -97,7 +118,9 @@ void testThreadPoolMetricsUnderLoad() throws Exception { ThreadPoolMetrics duringLoad = getThreadPoolMetrics(); assertThat("current threads during load", duringLoad.currentThreadCount(), greaterThanOrEqualTo(baseline.currentThreadCount())); - assertThat("busy threads during load", duringLoad.currentThreadsBusy(), greaterThanOrEqualTo(1)); + int numberOfConcurrentRequests = Math.min(activeRequests.get(), baseline.maxThreads()); + LOG.info("Number of active requests: " + numberOfConcurrentRequests); + assertThat("busy threads during load at least number of current active requests or max threads", duringLoad.currentThreadsBusy(), greaterThanOrEqualTo(numberOfConcurrentRequests)); assertThat("busy threads during load", duringLoad.currentThreadsBusy(), lessThanOrEqualTo(duringLoad.currentThreadCount())); assertThat("current threads during load", duringLoad.currentThreadCount(), lessThanOrEqualTo(duringLoad.maxThreads())); @@ -264,8 +287,8 @@ void testThreadPoolMetricsUnderSustainedLoad() throws Exception { private static File createDeployment() throws IOException { WebArchive war = ShrinkWrap.create(WebArchive.class, APP_NAME + ".war") - .addClass(TestServlet.class) - .addClass(SlowServlet.class); + .addClass(TestServlet.class) + .addClass(SlowServlet.class); File warFile = new File(System.getProperty("java.io.tmpdir"), APP_NAME + ".war"); war.as(ZipExporter.class).exportTo(warFile, true); @@ -278,21 +301,18 @@ private ThreadPoolMetrics getThreadPoolMetrics() { private ThreadPoolMetrics getThreadPoolMetrics(String listenerName) { // First, let's see what monitoring data is actually available - AsadminResult listResult = ASADMIN.exec("list", "*thread*"); - System.out.println("Available thread monitoring paths: " + listResult.getStdOut()); + AsadminResult listResult = ASADMIN.exec("list", "-m", "*thread-pool*"); + LOG.info("Available thread monitoring paths: " + listResult.getStdOut()); // Try to get metrics from the specified listener String basePath = "server.network." + listenerName + ".thread-pool"; - AsadminResult currentResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadcount"); - AsadminResult busyResult = ASADMIN.exec("get", "-m", basePath + ".currentthreadsbusy"); - AsadminResult maxResult = ASADMIN.exec("get", "-m", basePath + ".maxthreads"); - AsadminResult minResult = ASADMIN.exec("get", "-m", basePath + ".minthreads"); + AsadminResult result = ASADMIN.exec("get", "-m", basePath + ".*"); return new ThreadPoolMetrics( - extractMetric(currentResult.getStdOut(), "currentthreadcount"), - extractMetric(busyResult.getStdOut(), "currentthreadsbusy"), - extractMetric(minResult.getStdOut(), "minthreads"), - extractMetric(maxResult.getStdOut(), "maxthreads") + extractMetric(result.getStdOut(), "currentthreadcount"), + extractMetric(result.getStdOut(), "currentthreadsbusy"), + extractMetric(result.getStdOut(), "minthreads"), + extractMetric(result.getStdOut(), "maxthreads") ); } @@ -303,7 +323,7 @@ private int extractMetric(String output, String metric) { return Integer.parseInt(line.split("=")[1].trim()); } } - return -1000; + return -911000; } private ThreadPoolConfig getThreadPoolConfig() { @@ -316,8 +336,8 @@ private ThreadPoolConfig getThreadPoolConfig(String threadPoolName) { AsadminResult maxResult = ASADMIN.exec("get", basePath + ".max-thread-pool-size"); return new ThreadPoolConfig( - extractConfig(minResult.getStdOut(), "min-thread-pool-size"), - extractConfig(maxResult.getStdOut(), "max-thread-pool-size") + extractConfig(minResult.getStdOut(), "min-thread-pool-size"), + extractConfig(maxResult.getStdOut(), "max-thread-pool-size") ); } @@ -328,7 +348,7 @@ private int extractConfig(String output, String key) { return Integer.parseInt(line.split("=")[1].trim()); } } - return -1000; + return -911000; } private void makeRequest(String urlString) { @@ -346,9 +366,11 @@ private void makeRequest(String urlString) { } private record ThreadPoolMetrics(int currentThreadCount, int currentThreadsBusy, int minThreads, int maxThreads) { + } private record ThreadPoolConfig(int minThreads, int maxThreads) { + } @Test @@ -448,14 +470,11 @@ void testThreadPoolSizeDecrease() throws Exception { assertThat("current threads after decrease", afterDecrease.currentThreadCount(), greaterThanOrEqualTo(0)); assertThat("busy threads after decrease", afterDecrease.currentThreadsBusy(), greaterThanOrEqualTo(0)); - // Since we had 80 concurrent requests, current threads should still be high - // (threads don't disappear instantly when pool size is reduced) - assertThat("current threads after decrease", afterDecrease.currentThreadCount(), greaterThanOrEqualTo(10)); - executor.shutdown(); Thread.sleep(4000); // Wait for requests to complete ThreadPoolMetrics afterCompletion = getThreadPoolMetrics(); + assertThat("current threads after completion", afterCompletion.currentThreadsBusy(), equalTo(0)); assertThat("current threads after completion", afterCompletion.currentThreadCount(), greaterThanOrEqualTo(0)); assertThat("current threads after completion", afterCompletion.currentThreadCount(), lessThanOrEqualTo(10)); @@ -466,9 +485,12 @@ void testThreadPoolSizeDecrease() throws Exception { @Test void testThreadPoolSizeCycling() throws Exception { - ThreadPoolConfig initial = getThreadPoolConfig(); - int originalMinThreads = initial.minThreads(); - int originalMaxThreads = initial.maxThreads(); + ThreadPoolConfig initialConfig = getThreadPoolConfig(); + int originalMinThreads = initialConfig.minThreads(); + int originalMaxThreads = initialConfig.maxThreads(); + ThreadPoolMetrics metrics = getThreadPoolMetrics(); + + assertThat("max threads in pool matches config", metrics.maxThreads(), equalTo(initialConfig.minThreads)); try { int[] testSizes = {originalMaxThreads + 3, originalMaxThreads - 1, originalMaxThreads + 7, originalMaxThreads}; @@ -483,11 +505,11 @@ void testThreadPoolSizeCycling() throws Exception { ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + testSize); Thread.sleep(1500); - ThreadPoolMetrics metrics = getThreadPoolMetrics(); - assertThat("max threads (testSize=" + testSize + ")", metrics.maxThreads(), equalTo(testSize)); + metrics = getThreadPoolMetrics(); + assertThat("max threads (case " + i + ", testSize=" + testSize + ")", metrics.maxThreads(), equalTo(testSize)); int currentThreadsMaxSize = Math.max(testSize, originalMinThreads); - assertThat("current threads (testSize=" + testSize + ")", metrics.currentThreadCount(), lessThanOrEqualTo(currentThreadsMaxSize)); - assertThat("busy threads (testSize=" + testSize + ")", metrics.currentThreadsBusy(), lessThanOrEqualTo(metrics.currentThreadCount())); + assertThat("current threads (case " + i + ", testSize=" + testSize + ")", metrics.currentThreadCount(), lessThanOrEqualTo(currentThreadsMaxSize)); + assertThat("busy threads (case " + i + ", testSize=" + testSize + ")", metrics.currentThreadsBusy(), lessThanOrEqualTo(metrics.currentThreadCount())); } } finally { @@ -495,6 +517,39 @@ void testThreadPoolSizeCycling() throws Exception { } } + static boolean stop = false; + + @BeforeEach + void continueIfNotStop() { + assumeFalse(stop); + } + + @AfterEach + void verifyAdminListener() { + try { + ThreadPoolMetrics adminBaseline = getThreadPoolMetrics("admin-listener"); + + // Verify admin listener has valid baseline metrics + assertThat("admin listener current threads", adminBaseline.currentThreadCount(), greaterThanOrEqualTo(0)); + assertThat("admin listener max threads", adminBaseline.maxThreads(), greaterThan(0)); + assertThat("admin listener busy threads", adminBaseline.currentThreadsBusy(), greaterThanOrEqualTo(0)); + } catch (Exception e) { + stop = true; + throw e; + } + } + + @RegisterExtension + AfterTestExecutionCallback afterTestExecutionCallback = new AfterTestExecutionCallback() { + @Override + public void afterTestExecution(ExtensionContext context) throws Exception { + Optional exception = context.getExecutionException(); + if (exception.isPresent()) { // has exception + stop = true; + } + } + }; + @Test void testAdminListenerThreadPoolMetrics() throws Exception { // Get baseline metrics for admin-listener @@ -544,17 +599,17 @@ void testAdminListenerThreadPoolMetrics() throws Exception { @Test void testDualListenerThreadPoolMetrics() throws Exception { - // First create http-listener-2 if it doesn't exist + // First create http-listener-test if it doesn't exist AsadminResult createResult = ASADMIN.exec("create-http-listener", - "--listenerport=8081", "--listeneraddress=0.0.0.0", - "--defaultvs=server", "http-listener-2"); + "--listenerport=8081", "--listeneraddress=0.0.0.0", + "--defaultvs=server", "http-listener-test"); try { Thread.sleep(2000); // Allow listener to initialize // Get baseline metrics for both listeners ThreadPoolMetrics listener1Baseline = getThreadPoolMetrics("http-listener-1"); - ThreadPoolMetrics listener2Baseline = getThreadPoolMetrics("http-listener-2"); + ThreadPoolMetrics listener2Baseline = getThreadPoolMetrics("http-listener-test"); // Verify both listeners have valid baseline metrics assertThat("listener 1 current threads", listener1Baseline.currentThreadCount(), greaterThanOrEqualTo(0)); @@ -573,7 +628,7 @@ void testDualListenerThreadPoolMetrics() throws Exception { // Check metrics during dual load ThreadPoolMetrics listener1UnderLoad = getThreadPoolMetrics("http-listener-1"); - ThreadPoolMetrics listener2UnderLoad = getThreadPoolMetrics("http-listener-2"); + ThreadPoolMetrics listener2UnderLoad = getThreadPoolMetrics("http-listener-test"); // Both listeners should show activity assertThat("listener 1 current threads under load", listener1UnderLoad.currentThreadCount(), greaterThanOrEqualTo(0)); @@ -590,15 +645,15 @@ void testDualListenerThreadPoolMetrics() throws Exception { // Final metrics check ThreadPoolMetrics listener1Final = getThreadPoolMetrics("http-listener-1"); - ThreadPoolMetrics listener2Final = getThreadPoolMetrics("http-listener-2"); + ThreadPoolMetrics listener2Final = getThreadPoolMetrics("http-listener-test"); // Both should have valid final states assertThat("listener 1 final current threads", listener1Final.currentThreadCount(), greaterThanOrEqualTo(0)); assertThat("listener 2 final current threads", listener2Final.currentThreadCount(), greaterThanOrEqualTo(0)); } finally { - // Clean up http-listener-2 - ASADMIN.exec("delete-http-listener", "http-listener-2"); + // Clean up http-listener-test + ASADMIN.exec("delete-http-listener", "http-listener-test"); } } diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GlassfishNetworkListener.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GlassfishNetworkListener.java index 5fa6c749962..353636f64ec 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GlassfishNetworkListener.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GlassfishNetworkListener.java @@ -83,6 +83,11 @@ public NetworkListener getNetworkListener() { @Override public void stop() throws IOException { + stop(true); + } + + @Override + public void stop(boolean lastOne) throws IOException { ServiceLocator locator = grizzlyService.getServiceLocator(); IndexedFilter removeFilter = BuilderHelper.createNameAndContractFilter(Mapper.class.getName(), (address.toString() + port)); @@ -94,7 +99,13 @@ public void stop() throws IOException { config.commit(); - unregisterMonitoringStatsProviders(); + // Do not call unregisterMonitoringStatsProviders() + // - providers are shared by all listeners, they would be removed also for other listeners. + // We could add a rule to unregister only if there are no other listeners. + if (lastOne) { + unregisterMonitoringStatsProviders(); + } + super.stop(); } diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyProxy.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyProxy.java index 9a5d6013f18..2d173aa013f 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyProxy.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyProxy.java @@ -143,6 +143,14 @@ public void stop() throws IOException { grizzlyListener.stop(); } + /** + * Stops the Grizzly service and shared resources if last one. + */ + @Override + public void stop(boolean lastOne) throws IOException { + grizzlyListener.stop(lastOne); + } + @Override public void destroy() { grizzlyListener.destroy(); diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyService.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyService.java index d4e52c5e4f2..dcb4ba51817 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyService.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/GrizzlyService.java @@ -203,7 +203,8 @@ private NetworkProxy getNetworkProxy(String id) { public boolean removeNetworkProxy(NetworkProxy proxy) { if (proxy != null) { try { - proxy.stop(); + boolean lastOne = proxies.size() == 1; + proxy.stop(lastOne); } catch (IOException e) { LOGGER.log(WARNING, KernelLoggerInfo.grizzlyStopProxy, e); } diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/NetworkProxy.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/NetworkProxy.java index b31d525e817..2c31dccdeb8 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/NetworkProxy.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/NetworkProxy.java @@ -42,6 +42,14 @@ public interface NetworkProxy extends EndpointMapper{ */ void stop() throws IOException; + /** + * Stop the proxy and remove all shared resources if it's the last proxy. + * @param lastOne Whether it's the last proxy. + * {@code false} if other proxies still exist and depend on the shared resources + */ + default void stop(boolean lastOne) throws IOException { + stop(); + } /** * Start the proxy. diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java index 7e5fdbe35ac..904be0ecb12 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java @@ -19,6 +19,8 @@ import com.sun.enterprise.v3.services.impl.monitor.stats.ConnectionQueueStatsProvider; import com.sun.enterprise.v3.services.impl.monitor.stats.ThreadPoolStatsProvider; +import java.util.concurrent.atomic.AtomicInteger; + import org.glassfish.grizzly.threadpool.AbstractThreadPool; import org.glassfish.grizzly.threadpool.ThreadPoolConfig; import org.glassfish.grizzly.threadpool.ThreadPoolProbe; @@ -30,6 +32,7 @@ public class ThreadPoolMonitor implements ThreadPoolProbe { private final GrizzlyMonitoring grizzlyMonitoring; private final String monitoringId; + private AtomicInteger busyThreadCount = new AtomicInteger(0); public ThreadPoolMonitor(GrizzlyMonitoring grizzlyMonitoring, String monitoringId, ThreadPoolConfig config) { @@ -93,9 +96,11 @@ public void onMaxNumberOfThreadsEvent(AbstractThreadPool threadPool, int maxNumb @Override public void onTaskDequeueEvent(AbstractThreadPool threadPool, Runnable task) { + int busyCount = busyThreadCount.incrementAndGet(); grizzlyMonitoring.getThreadPoolProbeProvider().threadDispatchedFromPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), - Thread.currentThread().getId()); + Thread.currentThread().getId(), + busyCount); grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskDequeuedEvent( monitoringId, task.getClass().getName()); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( @@ -105,11 +110,13 @@ public void onTaskDequeueEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskCancelEvent(AbstractThreadPool threadPool, Runnable task) { + int busyCount = busyThreadCount.decrementAndGet(); // when dequeued task is cancelled - we have to "return" the thread, that // we marked as dispatched from the pool grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), - Thread.currentThread().getId()); + Thread.currentThread().getId(), + busyCount); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( monitoringId, threadPool.getConfig().getPoolName(), threadPool.getSize()); @@ -117,9 +124,11 @@ public void onTaskCancelEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskCompleteEvent(AbstractThreadPool threadPool, Runnable task) { + int busyCount = busyThreadCount.decrementAndGet(); grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), - Thread.currentThread().getId()); + Thread.currentThread().getId(), + busyCount); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( monitoringId, threadPool.getConfig().getPoolName(), threadPool.getSize()); diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java index 4d6d4d75c70..52feb47639a 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/probes/ThreadPoolProbeProvider.java @@ -68,14 +68,16 @@ public void maxNumberOfThreadsReachedEvent( public void threadDispatchedFromPoolEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, - @ProbeParam("threadId") long threadId) {} + @ProbeParam("threadId") long threadId, + @ProbeParam("busyThreadCount") long busyThreadCount) {} @Probe(name="threadReturnedToPoolEvent") public void threadReturnedToPoolEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, - @ProbeParam("threadId") long threadId) {} + @ProbeParam("threadId") long threadId, + @ProbeParam("busyThreadCount") long busyThreadCount) {} @Probe(name="setCurrentThreadCountEvent") public void setCurrentThreadCountEvent( diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java index 4628bb49f2e..bb3210195ad 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java @@ -143,10 +143,11 @@ public void threadReleasedEvent( public void threadDispatchedFromPoolEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, - @ProbeParam("threadId") long threadId) { + @ProbeParam("threadId") long threadId, + @ProbeParam("busyThreadCount") long busyThreadCount) { if (name.equals(monitoringId)) { - currentThreadsBusy.increment(); + currentThreadsBusy.setCount(busyThreadCount); } } @@ -154,11 +155,12 @@ public void threadDispatchedFromPoolEvent( public void threadReturnedToPoolEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, - @ProbeParam("threadId") long threadId) { + @ProbeParam("threadId") long threadId, + @ProbeParam("busyThreadCount") long busyThreadCount) { if (name.equals(monitoringId)) { totalExecutedTasksCount.increment(); - currentThreadsBusy.decrement(); + currentThreadsBusy.setCount(busyThreadCount); } } @@ -167,7 +169,6 @@ public void setCurrentThreadCountEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, @ProbeParam("currentThreadCount") int currentThreadCount) { - if (name.equals(monitoringId)) { this.currentThreadCount.setCount(currentThreadCount); } diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProviderGlobal.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProviderGlobal.java index cee2c0c0460..a8e925d9074 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProviderGlobal.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProviderGlobal.java @@ -76,19 +76,29 @@ public void threadReleasedEvent( public void threadDispatchedFromPoolEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, - @ProbeParam("threadId") long threadId) { + @ProbeParam("threadId") long threadId, + @ProbeParam("busyThreadCount") long busyThreadCount) { - currentThreadsBusy.increment(); + currentThreadsBusy.setCount(busyThreadCount); } @ProbeListener("glassfish:kernel:thread-pool:threadReturnedToPoolEvent") public void threadReturnedToPoolEvent( @ProbeParam("monitoringId") String monitoringId, @ProbeParam("threadPoolName") String threadPoolName, - @ProbeParam("threadId") long threadId) { + @ProbeParam("threadId") long threadId, + @ProbeParam("busyThreadCount") long busyThreadCount) { totalExecutedTasksCount.increment(); - currentThreadsBusy.decrement(); + currentThreadsBusy.setCount(busyThreadCount); + } + + @ProbeListener("glassfish:kernel:thread-pool:setCurrentThreadCountEvent") + public void setCurrentThreadCountEvent( + @ProbeParam("monitoringId") String monitoringId, + @ProbeParam("threadPoolName") String threadPoolName, + @ProbeParam("currentThreadCount") int currentThreadCount) { + this.currentThreadCount.setCount(currentThreadCount); } } diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java index b13a57fb185..54caecf074f 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java @@ -37,6 +37,8 @@ import org.glassfish.flashlight.client.ProbeHandle; import org.glassfish.flashlight.client.StatefulProbeClientInvoker; +import static java.util.logging.Level.FINE; + public class FlashlightProbe implements ProbeHandle, ProbeInfo{ @@ -148,11 +150,15 @@ public void fireProbe(Object[] params) { int sz = invokerList.size(); - for (int i=0; i Date: Tue, 28 Oct 2025 13:02:32 +0100 Subject: [PATCH 15/18] Improve iterating invokers in flashlight --- .../flashlight/provider/FlashlightProbe.java | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java index 54caecf074f..bc17912027f 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java @@ -148,17 +148,12 @@ public void fireProbe(Object[] params) { parent.fireProbe(params); } - int sz = invokerList.size(); + List invokers = new ArrayList<>(invokerList); - try { - for (int i=0; i fireProbeBefore(Object[] params) { probeInvokeStates.addAll(parentStates); } - int sz = invokerList.size(); + List invokers = new ArrayList<>(invokerList); - for (int i=0; i invokers = new ArrayList<>(invokerList); int stateIndex = -1; - for (int i=0; i= 0) - invoker.invokeOnException(states.get(stateIndex).getState(), exceptionValue); + statefulInvoker.invokeOnException(states.get(stateIndex).getState(), exceptionValue); } } } From a484822bb06052a99e562b13dc2f0b1a445976b7 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Tue, 28 Oct 2025 19:35:03 +0100 Subject: [PATCH 16/18] Reset thread pool monitoring stats to current values instead of 0 Also adds tests for it. Also adds support for a system property glassfish.remote, to run tests against a remote server instead of setting up a local (managed) one and stopping it after tests. This helps in debugging test scenarios. --- .../itest/tools/GlassFishTestEnvironment.java | 32 ++++++----- .../monitoring/ThreadPoolMonitoringTest.java | 57 +++++++++++++++++-- .../impl/monitor/ThreadPoolMonitor.java | 28 +++++---- .../impl/monitor/stats/ThreadPoolStats.java | 39 +++++++++++++ .../stats/ThreadPoolStatsProvider.java | 29 ++++++---- 5 files changed, 145 insertions(+), 40 deletions(-) create mode 100644 nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java diff --git a/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/GlassFishTestEnvironment.java b/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/GlassFishTestEnvironment.java index e1083229d54..60e65d24ace 100644 --- a/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/GlassFishTestEnvironment.java +++ b/appserver/itest-tools/src/main/java/org/glassfish/main/itest/tools/GlassFishTestEnvironment.java @@ -96,19 +96,21 @@ public class GlassFishTestEnvironment { private static HttpClient client; static { - LOG.log(Level.INFO, "Using basedir: {0}", BASEDIR); - LOG.log(Level.INFO, "Expected GlassFish directory: {0}", GF_ROOT); - changePassword(); - Thread hook = new Thread(() -> { - getAsadmin().exec(30_000, "stop-domain", "--kill", "--force"); - }); - Runtime.getRuntime().addShutdownHook(hook); - final int timeout = isStartDomainSuspendEnabled() - ? ASADMIN_START_DOMAIN_TIMEOUT_FOR_DEBUG : ASADMIN_START_DOMAIN_TIMEOUT; - // This is the absolutely first start - if it fails, all other starts will fail too. - // Note: --suspend implicitly enables --debug - assertThat(getAsadmin().exec(timeout,"start-domain", - isStartDomainSuspendEnabled() ? "--suspend" : "--debug"), asadminOK()); + if (!isGlassFishRunningRemotely()) { + LOG.log(Level.INFO, "Using basedir: {0}", BASEDIR); + LOG.log(Level.INFO, "Expected GlassFish directory: {0}", GF_ROOT); + changePassword(); + Thread hook = new Thread(() -> { + getAsadmin().exec(30_000, "stop-domain", "--kill", "--force"); + }); + Runtime.getRuntime().addShutdownHook(hook); + final int timeout = isStartDomainSuspendEnabled() + ? ASADMIN_START_DOMAIN_TIMEOUT_FOR_DEBUG : ASADMIN_START_DOMAIN_TIMEOUT; + // This is the absolutely first start - if it fails, all other starts will fail too. + // Note: --suspend implicitly enables --debug + assertThat(getAsadmin().exec(timeout,"start-domain", + isStartDomainSuspendEnabled() ? "--suspend" : "--debug"), asadminOK()); + } } @@ -428,6 +430,10 @@ private static boolean isStartDomainSuspendEnabled() { return envValue == null ? Boolean.getBoolean("glassfish.suspend") : Boolean.parseBoolean(envValue); } + public static boolean isGlassFishRunningRemotely() { + final String envValue = System.getenv("GLASSFISH_REMOTE"); + return envValue == null ? Boolean.getBoolean("glassfish.remote") : Boolean.parseBoolean(envValue); + } private static T doIO(IOSupplier action) { try { diff --git a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java index 5352349fc1d..e24c13bc231 100644 --- a/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java +++ b/appserver/tests/application/src/test/java/org/glassfish/main/test/app/monitoring/ThreadPoolMonitoringTest.java @@ -43,6 +43,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.jupiter.api.Assumptions.assumeFalse; @@ -80,12 +81,6 @@ static void undeployApp() { @Test void testThreadPoolMetricsUnderLoad() throws Exception { - AsadminResult createResult = ASADMIN.exec("create-http-listener", - "--listenerport=8081", "--listeneraddress=0.0.0.0", - "--defaultvs=server", "http-listener-test"); -// ASADMIN.exec("delete-http-listener", "http-listener-test"); - - // Get baseline metrics ThreadPoolMetrics baseline = getThreadPoolMetrics(); assertThat("baseline current thread count", baseline.currentThreadCount(), greaterThanOrEqualTo(0)); @@ -700,4 +695,54 @@ void testThreadPoolSizeUnderLoad() throws Exception { ASADMIN.exec("set", "configs.config.server-config.thread-pools.thread-pool.http-thread-pool.max-thread-pool-size=" + originalMaxThreads); } } + + @Test + void testThreadPoolMonitoringEnableDisable() throws Exception { + try { + // 1. Disable thread-pool monitoring + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=OFF"); + Thread.sleep(1000); // Allow monitoring to be disabled + + // 2. Schedule 3 long-running requests (5 seconds each) + ExecutorService executor = Executors.newFixedThreadPool(3); + CompletableFuture[] futures = new CompletableFuture[3]; + + for (int i = 0; i < 3; i++) { + futures[i] = CompletableFuture.runAsync(() -> { + try { + HttpURLConnection conn = GlassFishTestEnvironment.openConnection(8080, "/threadpool-test/slow?delay=10000"); + conn.setRequestMethod("GET"); + conn.getResponseCode(); + conn.disconnect(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, executor); + } + + // 3. Enable thread-pool monitoring while requests are running + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH"); + Thread.sleep(500); // Allow monitoring to be enabled + + // 4. Verify thread metrics show running requests + ThreadPoolMetrics metrics = getThreadPoolMetrics(); + + assertThat("current thread count with monitoring enabled", metrics.currentThreadCount(), greaterThan(0)); + assertThat("busy threads should reflect running requests", metrics.currentThreadsBusy(), greaterThanOrEqualTo(3)); + assertThat("busy threads should not exceed current threads", metrics.currentThreadsBusy(), lessThanOrEqualTo(metrics.currentThreadCount())); + + // Wait for requests to complete + CompletableFuture.allOf(futures).get(10, TimeUnit.SECONDS); + executor.shutdown(); + + // Verify metrics after requests complete + Thread.sleep(1000); + ThreadPoolMetrics afterCompletion = getThreadPoolMetrics(); + assertThat("busy threads after completion", afterCompletion.currentThreadsBusy(), lessThan(metrics.currentThreadsBusy())); + + } finally { + // Ensure monitoring is re-enabled for other tests + ASADMIN.exec("set", "configs.config.server-config.monitoring-service.module-monitoring-levels.thread-pool=HIGH"); + } + } } diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java index 904be0ecb12..b38d91dd7d8 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java @@ -17,10 +17,9 @@ package com.sun.enterprise.v3.services.impl.monitor; import com.sun.enterprise.v3.services.impl.monitor.stats.ConnectionQueueStatsProvider; +import com.sun.enterprise.v3.services.impl.monitor.stats.ThreadPoolStats; import com.sun.enterprise.v3.services.impl.monitor.stats.ThreadPoolStatsProvider; -import java.util.concurrent.atomic.AtomicInteger; - import org.glassfish.grizzly.threadpool.AbstractThreadPool; import org.glassfish.grizzly.threadpool.ThreadPoolConfig; import org.glassfish.grizzly.threadpool.ThreadPoolProbe; @@ -32,18 +31,19 @@ public class ThreadPoolMonitor implements ThreadPoolProbe { private final GrizzlyMonitoring grizzlyMonitoring; private final String monitoringId; - private AtomicInteger busyThreadCount = new AtomicInteger(0); + private final ThreadPoolStats stats; public ThreadPoolMonitor(GrizzlyMonitoring grizzlyMonitoring, String monitoringId, ThreadPoolConfig config) { this.grizzlyMonitoring = grizzlyMonitoring; this.monitoringId = monitoringId; + this.stats = new ThreadPoolStats(config); if (grizzlyMonitoring != null) { final ThreadPoolStatsProvider threadPoolStatsProvider = grizzlyMonitoring.getThreadPoolStatsProvider(monitoringId); if (threadPoolStatsProvider != null) { - threadPoolStatsProvider.setStatsObject(config); + threadPoolStatsProvider.setStatsObject(stats); threadPoolStatsProvider.reset(); } @@ -66,6 +66,7 @@ public void onThreadPoolStopEvent(AbstractThreadPool threadPool) { @Override public void onThreadAllocateEvent(AbstractThreadPool threadPool, Thread thread) { + stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getThreadPoolProbeProvider().threadAllocatedEvent( monitoringId, threadPool.getConfig().getPoolName(), thread.getId()); @@ -76,6 +77,7 @@ public void onThreadAllocateEvent(AbstractThreadPool threadPool, Thread thread) @Override public void onThreadReleaseEvent(AbstractThreadPool threadPool, Thread thread) { + stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getThreadPoolProbeProvider().threadReleasedEvent( monitoringId, threadPool.getConfig().getPoolName(), thread.getId()); @@ -86,6 +88,7 @@ public void onThreadReleaseEvent(AbstractThreadPool threadPool, Thread thread) { @Override public void onMaxNumberOfThreadsEvent(AbstractThreadPool threadPool, int maxNumberOfThreads) { + stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getThreadPoolProbeProvider().maxNumberOfThreadsReachedEvent( monitoringId, threadPool.getConfig().getPoolName(), maxNumberOfThreads); @@ -96,11 +99,12 @@ public void onMaxNumberOfThreadsEvent(AbstractThreadPool threadPool, int maxNumb @Override public void onTaskDequeueEvent(AbstractThreadPool threadPool, Runnable task) { - int busyCount = busyThreadCount.incrementAndGet(); + stats.currentBusyThreadCount++; + stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getThreadPoolProbeProvider().threadDispatchedFromPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId(), - busyCount); + stats.currentBusyThreadCount); grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskDequeuedEvent( monitoringId, task.getClass().getName()); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( @@ -110,13 +114,14 @@ public void onTaskDequeueEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskCancelEvent(AbstractThreadPool threadPool, Runnable task) { - int busyCount = busyThreadCount.decrementAndGet(); + stats.currentBusyThreadCount--; + stats.currentThreadCount = threadPool.getSize(); // when dequeued task is cancelled - we have to "return" the thread, that // we marked as dispatched from the pool grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId(), - busyCount); + stats.currentBusyThreadCount); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( monitoringId, threadPool.getConfig().getPoolName(), threadPool.getSize()); @@ -124,11 +129,12 @@ public void onTaskCancelEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskCompleteEvent(AbstractThreadPool threadPool, Runnable task) { - int busyCount = busyThreadCount.decrementAndGet(); + stats.currentBusyThreadCount--; + stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId(), - busyCount); + stats.currentBusyThreadCount); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( monitoringId, threadPool.getConfig().getPoolName(), threadPool.getSize()); @@ -136,6 +142,7 @@ public void onTaskCompleteEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskQueueEvent(AbstractThreadPool threadPool, Runnable task) { + stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskQueuedEvent( monitoringId, task.getClass().getName()); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( @@ -145,6 +152,7 @@ public void onTaskQueueEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskQueueOverflowEvent(AbstractThreadPool threadPool) { + stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskQueueOverflowEvent( monitoringId); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java new file mode 100644 index 00000000000..24177c52967 --- /dev/null +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2025 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0, which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the + * Eclipse Public License v. 2.0 are satisfied: GNU General Public License, + * version 2 with the GNU Classpath Exception, which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + */ +package com.sun.enterprise.v3.services.impl.monitor.stats; + +import org.glassfish.grizzly.threadpool.ThreadPoolConfig; + +/** + * An object to hold thread pool statistics. Shared by ThreadPoolMonitor and stats providers (probe listeners), + * as some stats should be preserved even if the listeners is down. + * For example, thread counters, if there's no reliable way to find out the actual thread numbers from the thread pool (like busy threads). + * + * @author Ondro Mihalyi + */ +public class ThreadPoolStats { + + public ThreadPoolConfig threadPoolConfig; + + public long currentThreadCount; + + public long currentBusyThreadCount; + + public ThreadPoolStats(ThreadPoolConfig threadPoolConfig) { + this.threadPoolConfig = threadPoolConfig; + } + +} diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java index bb3210195ad..d0b4271abbc 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java @@ -45,7 +45,7 @@ public class ThreadPoolStatsProvider implements StatsProvider { protected final CountStatisticImpl currentThreadCount = new CountStatisticImpl("CurrentThreadCount", "count", "Provides the number of request processing threads currently in the listener thread pool"); protected final CountStatisticImpl currentThreadsBusy = new CountStatisticImpl("CurrentThreadsBusy", "count", "Provides the number of request processing threads currently in use in the listener thread pool serving requests"); - protected volatile ThreadPoolConfig threadPoolConfig; + protected volatile ThreadPoolStats threadPoolStats; public ThreadPoolStatsProvider(String name) { this.name = name; @@ -53,15 +53,15 @@ public ThreadPoolStatsProvider(String name) { @Override public Object getStatsObject() { - return threadPoolConfig; + return threadPoolStats; } @Override public void setStatsObject(Object object) { - if (object instanceof ThreadPoolConfig) { - threadPoolConfig = (ThreadPoolConfig) object; + if (object instanceof ThreadPoolStats) { + threadPoolStats = (ThreadPoolStats) object; } else { - threadPoolConfig = null; + threadPoolStats = null; } } @@ -86,12 +86,18 @@ public CountStatistic getTotalExecutedTasksCount() { @ManagedAttribute(id = "currentthreadcount") @Description("Provides the number of request processing threads currently in the listener thread pool") public CountStatistic getCurrentThreadCount() { + if (threadPoolStats != null) { + currentThreadCount.setCount(threadPoolStats.currentThreadCount); + } return currentThreadCount; } @ManagedAttribute(id = "currentthreadsbusy") @Description("Provides the number of request processing threads currently in use in the listener thread pool serving requests.") public CountStatistic getCurrentThreadsBusy() { + if (threadPoolStats != null) { + currentThreadsBusy.setCount(threadPoolStats.currentBusyThreadCount); + } return currentThreadsBusy; } @@ -176,13 +182,14 @@ public void setCurrentThreadCountEvent( @Reset public void reset() { - if (threadPoolConfig != null) { - maxThreadsCount.setCount(threadPoolConfig.getMaxPoolSize()); - coreThreadsCount.setCount(threadPoolConfig.getCorePoolSize()); - currentThreadCount.setCount(0); - currentThreadsBusy.setCount(0); + if (threadPoolStats != null) { + if (threadPoolStats.threadPoolConfig != null) { + maxThreadsCount.setCount(threadPoolStats.threadPoolConfig.getMaxPoolSize()); + coreThreadsCount.setCount(threadPoolStats.threadPoolConfig.getCorePoolSize()); + } + currentThreadCount.setCount(threadPoolStats.currentThreadCount); + currentThreadsBusy.setCount(threadPoolStats.currentBusyThreadCount); } - totalExecutedTasksCount.setCount(0); } } From 1db1926b357ffa11f7cb1148d196e4acb97ef7d3 Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Wed, 29 Oct 2025 01:09:39 +0100 Subject: [PATCH 17/18] Fix race conditions in thread pool monitoring Introduced by a previous "fix" :) --- appserver/extras/embedded/pom.xml | 2 +- .../v3/services/impl/monitor/ThreadPoolMonitor.java | 12 ++++++------ .../services/impl/monitor/stats/ThreadPoolStats.java | 6 ++++-- .../impl/monitor/stats/ThreadPoolStatsProvider.java | 5 ++--- .../flashlight/provider/FlashlightProbe.java | 2 -- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/appserver/extras/embedded/pom.xml b/appserver/extras/embedded/pom.xml index 0902b2f5aa0..8dd40eb9927 100644 --- a/appserver/extras/embedded/pom.xml +++ b/appserver/extras/embedded/pom.xml @@ -29,7 +29,7 @@ pom GlassFish Embedded modules - + java.base/java.lang java.base/java.io java.base/java.util java.base/sun.nio.fs java.base/sun.net.www.protocol.jrt java.naming/javax.naming.spi java.rmi/sun.rmi.transport jdk.management/com.sun.management.internal java.base/jdk.internal.vm.annotation java.naming/com.sun.jndi.ldap java.base/jdk.internal.vm.annotation java.base/jdk.internal.loader diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java index b38d91dd7d8..3a26367be4a 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/ThreadPoolMonitor.java @@ -99,12 +99,12 @@ public void onMaxNumberOfThreadsEvent(AbstractThreadPool threadPool, int maxNumb @Override public void onTaskDequeueEvent(AbstractThreadPool threadPool, Runnable task) { - stats.currentBusyThreadCount++; + long currentBusyThreadCount = stats.currentBusyThreadCount.incrementAndGet(); stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getThreadPoolProbeProvider().threadDispatchedFromPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId(), - stats.currentBusyThreadCount); + currentBusyThreadCount); grizzlyMonitoring.getConnectionQueueProbeProvider().onTaskDequeuedEvent( monitoringId, task.getClass().getName()); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( @@ -114,14 +114,14 @@ public void onTaskDequeueEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskCancelEvent(AbstractThreadPool threadPool, Runnable task) { - stats.currentBusyThreadCount--; + long currentBusyThreadCount = stats.currentBusyThreadCount.decrementAndGet(); stats.currentThreadCount = threadPool.getSize(); // when dequeued task is cancelled - we have to "return" the thread, that // we marked as dispatched from the pool grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId(), - stats.currentBusyThreadCount); + currentBusyThreadCount); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( monitoringId, threadPool.getConfig().getPoolName(), threadPool.getSize()); @@ -129,12 +129,12 @@ public void onTaskCancelEvent(AbstractThreadPool threadPool, Runnable task) { @Override public void onTaskCompleteEvent(AbstractThreadPool threadPool, Runnable task) { - stats.currentBusyThreadCount--; + long currentBusyThreadCount = stats.currentBusyThreadCount.decrementAndGet(); stats.currentThreadCount = threadPool.getSize(); grizzlyMonitoring.getThreadPoolProbeProvider().threadReturnedToPoolEvent( monitoringId, threadPool.getConfig().getPoolName(), Thread.currentThread().getId(), - stats.currentBusyThreadCount); + currentBusyThreadCount); grizzlyMonitoring.getThreadPoolProbeProvider().setCurrentThreadCountEvent( monitoringId, threadPool.getConfig().getPoolName(), threadPool.getSize()); diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java index 24177c52967..83a0371f037 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStats.java @@ -15,6 +15,8 @@ */ package com.sun.enterprise.v3.services.impl.monitor.stats; +import java.util.concurrent.atomic.AtomicLong; + import org.glassfish.grizzly.threadpool.ThreadPoolConfig; /** @@ -28,9 +30,9 @@ public class ThreadPoolStats { public ThreadPoolConfig threadPoolConfig; - public long currentThreadCount; + public volatile long currentThreadCount; - public long currentBusyThreadCount; + public AtomicLong currentBusyThreadCount = new AtomicLong(); public ThreadPoolStats(ThreadPoolConfig threadPoolConfig) { this.threadPoolConfig = threadPoolConfig; diff --git a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java index d0b4271abbc..0d213ece593 100644 --- a/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java +++ b/nucleus/core/kernel/src/main/java/com/sun/enterprise/v3/services/impl/monitor/stats/ThreadPoolStatsProvider.java @@ -25,7 +25,6 @@ import org.glassfish.gmbal.Description; import org.glassfish.gmbal.ManagedAttribute; import org.glassfish.gmbal.ManagedObject; -import org.glassfish.grizzly.threadpool.ThreadPoolConfig; /** * Thread Pool statistics @@ -96,7 +95,7 @@ public CountStatistic getCurrentThreadCount() { @Description("Provides the number of request processing threads currently in use in the listener thread pool serving requests.") public CountStatistic getCurrentThreadsBusy() { if (threadPoolStats != null) { - currentThreadsBusy.setCount(threadPoolStats.currentBusyThreadCount); + currentThreadsBusy.setCount(threadPoolStats.currentBusyThreadCount.get()); } return currentThreadsBusy; } @@ -188,7 +187,7 @@ public void reset() { coreThreadsCount.setCount(threadPoolStats.threadPoolConfig.getCorePoolSize()); } currentThreadCount.setCount(threadPoolStats.currentThreadCount); - currentThreadsBusy.setCount(threadPoolStats.currentBusyThreadCount); + currentThreadsBusy.setCount(threadPoolStats.currentBusyThreadCount.get()); } totalExecutedTasksCount.setCount(0); } diff --git a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java index bc17912027f..5dc37c30ac8 100644 --- a/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java +++ b/nucleus/flashlight/framework/src/main/java/org/glassfish/flashlight/provider/FlashlightProbe.java @@ -37,8 +37,6 @@ import org.glassfish.flashlight.client.ProbeHandle; import org.glassfish.flashlight.client.StatefulProbeClientInvoker; -import static java.util.logging.Level.FINE; - public class FlashlightProbe implements ProbeHandle, ProbeInfo{ From dc4c6e74adf6ee185ae6e7617a5aeacc53445f9a Mon Sep 17 00:00:00 2001 From: Ondro Mihalyi Date: Wed, 29 Oct 2025 14:39:59 +0100 Subject: [PATCH 18/18] Capture errors in github workflows if build fails --- .github/workflows/build-linux.yml | 36 ++++++++++++++++++++++++- .github/workflows/build-macos.yml | 41 +++++++++++++++++++++++++++-- .github/workflows/build-windows.yml | 37 +++++++++++++++++++++++++- 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 422449a0536..75aac07d2a0 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -18,9 +18,43 @@ jobs: - name: List ports run: ss -tulpn - name: Build with Maven + id: maven-build # qa skips documentation - we check it on Jenkins CI # We skip checkstyle too - we check it on Jenkins CI - run: mvn -B -e -ntp install -Pstaging -Pqa '-Dcheckstyle.skip=true' + run: | + set -o pipefail + mvn -B -e -ntp install -Pstaging -Pqa '-Dcheckstyle.skip=true' 2>&1 | tee build.log + - name: Extract build failure info + if: failure() && steps.maven-build.outcome == 'failure' + run: | + echo "## Build Failure Summary" >> $GITHUB_STEP_SUMMARY + echo "### Last 100 lines of Maven output:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tail -100 build.log >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo "### Error Analysis:" >> $GITHUB_STEP_SUMMARY + if grep -q "BUILD FAILURE" build.log; then + echo "- ❌ Maven build failed" >> $GITHUB_STEP_SUMMARY + # Extract the failure reason + echo "### Failure Reason:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -A 10 "BUILD FAILURE" build.log | head -15 >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + if grep -q "COMPILATION ERROR" build.log; then + echo "- ❌ Compilation errors detected" >> $GITHUB_STEP_SUMMARY + fi + if grep -q "Tests run:.*Failures: [1-9]" build.log; then + echo "- ❌ Test failures detected" >> $GITHUB_STEP_SUMMARY + fi + - name: Upload build log + uses: actions/upload-artifact@v4 + if: failure() + with: + name: build-log + path: build.log + retention-days: 3 - name: Upload server logs uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index c203ba4a969..f27ad4e05e9 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -16,11 +16,48 @@ jobs: distribution: 'temurin' cache: maven - name: Build with Maven + id: maven-build # qa skips documentation - we check it on Jenkins CI # We skip checkstyle too - we check it on Jenkins CI - run: mvn -B -e -ntp install -Pstaging -Pqa '-Dcheckstyle.skip=true' -Pfastest + run: | + set -o pipefail + mvn -B -e -ntp install -Pstaging -Pqa '-Dcheckstyle.skip=true' -Pfastest 2>&1 | tee build.log - name: Test Starts - run: mvn -B -e -ntp install -Pstaging '-Dcheckstyle.skip=true' -pl :glassfish-itest-tools + id: test-starts + run: | + set -o pipefail + mvn -B -e -ntp install -Pstaging '-Dcheckstyle.skip=true' -pl :glassfish-itest-tools 2>&1 | tee -a build.log + - name: Extract build failure info + if: failure() && (steps.maven-build.outcome == 'failure' || steps.test-starts.outcome == 'failure') + run: | + echo "## Build Failure Summary" >> $GITHUB_STEP_SUMMARY + echo "### Last 100 lines of Maven output:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tail -100 build.log >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo "### Error Analysis:" >> $GITHUB_STEP_SUMMARY + if grep -q "BUILD FAILURE" build.log; then + echo "- ❌ Maven build failed" >> $GITHUB_STEP_SUMMARY + # Extract the failure reason + echo "### Failure Reason:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -A 10 "BUILD FAILURE" build.log | head -15 >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + if grep -q "COMPILATION ERROR" build.log; then + echo "- ❌ Compilation errors detected" >> $GITHUB_STEP_SUMMARY + fi + if grep -q "Tests run:.*Failures: [1-9]" build.log; then + echo "- ❌ Test failures detected" >> $GITHUB_STEP_SUMMARY + fi + - name: Upload build log + uses: actions/upload-artifact@v4 + if: failure() + with: + name: build-log + path: build.log + retention-days: 3 - name: Upload server logs uses: actions/upload-artifact@v4 if: failure() diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 1e80a4c04fd..87bcd766b44 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -16,9 +16,44 @@ jobs: distribution: 'temurin' cache: maven - name: Build with Maven + id: maven-build # qa skips documentation - we check it on Jenkins CI # We skip checkstyle too - we check it on Jenkins CI - run: mvn -B -e -ntp install -Pstaging -Pqa '-Dcheckstyle.skip=true' + run: | + mvn -B -e -ntp install -Pstaging -Pqa '-Dcheckstyle.skip=true' 2>&1 | Tee-Object build.log + - name: Extract build failure info + if: failure() && steps.maven-build.outcome == 'failure' + shell: pwsh + run: | + echo "## Build Failure Summary" >> $env:GITHUB_STEP_SUMMARY + echo "### Last 100 lines of Maven output:" >> $env:GITHUB_STEP_SUMMARY + echo '```' >> $env:GITHUB_STEP_SUMMARY + Get-Content build.log -Tail 100 >> $env:GITHUB_STEP_SUMMARY + echo '```' >> $env:GITHUB_STEP_SUMMARY + + echo "### Error Analysis:" >> $env:GITHUB_STEP_SUMMARY + $content = Get-Content build.log -Raw + if ($content -match "BUILD FAILURE") { + echo "- ❌ Maven build failed" >> $env:GITHUB_STEP_SUMMARY + # Extract the failure reason + echo "### Failure Reason:" >> $env:GITHUB_STEP_SUMMARY + echo '```' >> $env:GITHUB_STEP_SUMMARY + ($content | Select-String -Pattern "BUILD FAILURE" -Context 0,10).Context.PostContext[0..14] -join "`n" >> $env:GITHUB_STEP_SUMMARY + echo '```' >> $env:GITHUB_STEP_SUMMARY + } + if ($content -match "COMPILATION ERROR") { + echo "- ❌ Compilation errors detected" >> $env:GITHUB_STEP_SUMMARY + } + if ($content -match "Tests run:.*Failures: [1-9]") { + echo "- ❌ Test failures detected" >> $env:GITHUB_STEP_SUMMARY + } + - name: Upload build log + uses: actions/upload-artifact@v4 + if: failure() + with: + name: build-log + path: build.log + retention-days: 3 - name: Upload server logs uses: actions/upload-artifact@v4 if: failure()