From db67d8e1976bf27c28b2f1ddfbc36f6c89bcf929 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 11 Aug 2025 16:16:54 -0400 Subject: [PATCH 01/35] Debug freezing test on CI. --- .gitlab/collect_reports.sh | 1 + .../groovy/StructuredConcurrencyTest.groovy | 88 +++++++++++++++++-- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/.gitlab/collect_reports.sh b/.gitlab/collect_reports.sh index 83c3d99d38a..1b30beb6a6e 100755 --- a/.gitlab/collect_reports.sh +++ b/.gitlab/collect_reports.sh @@ -62,6 +62,7 @@ function process_reports () { cp -r workspace/$project_to_save/build/reports/* $report_path/ 2>/dev/null || true cp workspace/$project_to_save/build/hs_err_pid*.log $report_path/ 2>/dev/null || true cp workspace/$project_to_save/build/javacore*.txt $report_path/ 2>/dev/null || true + cp workspace/$project_to_save/build/*.* $report_path/ 2>/dev/null || true fi } diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 0c0280afbaf..3269ac5ed4c 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -1,14 +1,46 @@ +import com.sun.management.HotSpotDiagnosticMXBean import datadog.trace.agent.test.AgentTestRunner import datadog.trace.api.Trace +import javax.management.MBeanServer +import java.lang.management.ManagementFactory import java.util.concurrent.Callable +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture import java.util.concurrent.StructuredTaskScope +import java.util.concurrent.TimeUnit import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace import static java.time.Instant.now class StructuredConcurrencyTest extends AgentTestRunner { + ThreadDumpLogger threadDumpLogger + + def setup() { + File reportDir = new File("build") + String fullPath = reportDir.absolutePath.replace("dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent") + + reportDir = new File(fullPath) + if (!reportDir.exists()) { + println("Folder not found: " + fullPath) + reportDir.mkdirs() + } else println("Folder found: " + fullPath) + + // Use the current feature name as the test name + String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}" + + threadDumpLogger = new ThreadDumpLogger(testName, reportDir) + threadDumpLogger.start() + } + + def cleanup() { + threadDumpLogger.stop() + } + + /** * Tests the structured task scope with a single task. */ @@ -20,12 +52,12 @@ class StructuredConcurrencyTest extends AgentTestRunner { when: runUnderTrace("parent") { def task = taskScope.fork(new Callable() { - @Trace(operationName = "child") - @Override - Boolean call() throws Exception { - return true - } - }) + @Trace(operationName = "child") + @Override + Boolean call() throws Exception { + return true + } + }) taskScope.joinUntil(now() + 10) // Wait for 10 seconds at maximum result = task.get() } @@ -164,4 +196,48 @@ class StructuredConcurrencyTest extends AgentTestRunner { } } } + // 🔒 Private helper class for thread dump logging + private static class ThreadDumpLogger { + private final String testName + private final File outputDir + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor() + private ScheduledFuture task + + ThreadDumpLogger(String testName, File outputDir) { + this.testName = testName + this.outputDir = outputDir + } + + void start() { + task = scheduler.scheduleAtFixedRate({ + heapDump("test_1") + + def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") + try (def writer = new FileWriter(reportFile)) { + writer.write("=== Test: ${testName} ===\n") + writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") + Thread.getAllStackTraces().each { thread, stack -> + writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") + stack.each { writer.write("\tat ${it}\n") } + } + writer.write("==============================================\n") + } + + heapDump("test_2") + }, 10000, 60000, TimeUnit.MILLISECONDS) + } + + void heapDump(String kind) { + def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath + MBeanServer server = ManagementFactory.getPlatformMBeanServer() + HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) + mxBean.dumpHeap(heapDumpFile, true) + } + + void stop() { + task?.cancel(false) + scheduler.shutdownNow() + } + } } From 025e232d7f92241b93c48ce2120bb49de88c3de2 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 12 Aug 2025 16:58:38 -0400 Subject: [PATCH 02/35] Trying to reproduce. --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 3269ac5ed4c..dd46734cd35 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 10000, 60000, TimeUnit.MILLISECONDS) + }, 10001, 60000, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From b62fc0306ad1ca130d370b47e9e71f086655185b Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 12 Aug 2025 18:39:46 -0400 Subject: [PATCH 03/35] Trying to reproduce. --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index dd46734cd35..b6b93bc6b59 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 10001, 60000, TimeUnit.MILLISECONDS) + }, 10002, 60001, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From 835f3130a1cb37dd0817de5f3b8c51202e06b17e Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 13 Aug 2025 16:19:58 -0400 Subject: [PATCH 04/35] Trying to reproduce. --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index b6b93bc6b59..d8cb6e46d37 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 10002, 60001, TimeUnit.MILLISECONDS) + }, 10003, 60001, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From a1fae6ac4f8d6ebcafb75461daf556ec706d479f Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 14 Aug 2025 15:10:45 -0400 Subject: [PATCH 05/35] Updated flaky tests. --- .../datadog/trace/agent/test/base/HttpClientTest.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy index 96383049879..5ef385bf1a9 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy @@ -275,6 +275,7 @@ abstract class HttpClientTest extends VersionedNamingTestBase { body = (1..10000).join(" ") } + @Flaky(suites = ["ApacheHttpAsyncClient5NamingV0Test"]) def "basic #method request with parent"() { when: def status = runUnderTrace("parent") { @@ -308,7 +309,7 @@ abstract class HttpClientTest extends VersionedNamingTestBase { body = (1..10000).join(" ") } - @Flaky(suites = ["ApacheHttpAsyncClient5Test"]) + @Flaky(suites = ["ApacheHttpAsyncClient5NamingV0Test"]) def "server error request with parent"() { setup: def uri = server.address.resolve("/error") @@ -346,7 +347,7 @@ abstract class HttpClientTest extends VersionedNamingTestBase { "POST" | _ } - @Flaky(suites = ["ApacheHttpAsyncClient5Test", "ApacheHttpAsyncClient5NamingV0Test"]) + @Flaky(suites = ["ApacheHttpAsyncClient5NamingV0Test"]) def "client error request with parent"() { setup: def uri = server.address.resolve("/secured") @@ -415,7 +416,6 @@ abstract class HttpClientTest extends VersionedNamingTestBase { method = "HEAD" } - @Flaky(suites = ["ApacheHttpAsyncClient5Test"]) def "trace request without propagation"() { when: injectSysConfig(HTTP_CLIENT_HOST_SPLIT_BY_DOMAIN, "$renameService") From 60deecd945d2530a44f32ecf48522587afd21fc4 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 14 Aug 2025 15:37:25 -0400 Subject: [PATCH 06/35] Trying to reproduce. --- .../groovy/StructuredConcurrencyTest.groovy | 2 +- .../test/groovy/Lettuce4ClientTestBase.groovy | 83 +++++++++++++++++-- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index d8cb6e46d37..0d60304ad91 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 10003, 60001, TimeUnit.MILLISECONDS) + }, 10005, 60005, TimeUnit.MILLISECONDS) } void heapDump(String kind) { diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 6d178e33622..7e1edfea332 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -3,11 +3,19 @@ import com.lambdaworks.redis.RedisClient import com.lambdaworks.redis.api.StatefulConnection import com.lambdaworks.redis.api.async.RedisAsyncCommands import com.lambdaworks.redis.api.sync.RedisCommands +import com.sun.management.HotSpotDiagnosticMXBean import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.agent.test.utils.PortUtils import redis.embedded.RedisServer import spock.lang.Shared +import javax.management.MBeanServer +import java.lang.management.ManagementFactory +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit + import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { @@ -32,6 +40,8 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { @Shared RedisServer redisServer + ThreadDumpLogger threadDumpLogger + @Shared Map testHashMap = [ firstname: "John", @@ -53,14 +63,30 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { embeddedDbUri = "redis://" + dbAddr redisServer = RedisServer.newRedisServer() - // bind to localhost to avoid firewall popup - .setting("bind " + HOST) - // set max memory to avoid problems in CI - .setting("maxmemory 128M") - .port(port).build() + // bind to localhost to avoid firewall popup + .setting("bind " + HOST) + // set max memory to avoid problems in CI + .setting("maxmemory 128M") + .port(port).build() } def setup() { + File reportDir = new File("build") + String fullPath = reportDir.absolutePath.replace("dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent") + + reportDir = new File(fullPath) + if (!reportDir.exists()) { + println("Folder not found: " + fullPath) + reportDir.mkdirs() + } else println("Folder found: " + fullPath) + + // Use the current feature name as the test name + String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}" + + threadDumpLogger = new ThreadDumpLogger(testName, reportDir) + threadDumpLogger.start() + redisServer.start() redisClient = RedisClient.create(embeddedDbUri) @@ -79,8 +105,55 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } def cleanup() { + threadDumpLogger.stop() + connection.close() redisClient.shutdown() redisServer.stop() } + + // 🔒 Private helper class for thread dump logging + private static class ThreadDumpLogger { + private final String testName + private final File outputDir + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor() + private ScheduledFuture task + + ThreadDumpLogger(String testName, File outputDir) { + this.testName = testName + this.outputDir = outputDir + } + + void start() { + // new File(outputDir, "${System.currentTimeMillis()}-start-mark.txt") << testName + + task = scheduler.scheduleAtFixedRate({ + heapDump("test") + + def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") + try (def writer = new FileWriter(reportFile)) { + writer.write("=== Test: ${testName} ===\n") + writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") + Thread.getAllStackTraces().each { thread, stack -> + writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") + stack.each { writer.write("\tat ${it}\n") } + } + writer.write("==============================================\n") + } + }, 10003, 60000, TimeUnit.MILLISECONDS) + } + + void heapDump(String kind) { + def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath + MBeanServer server = ManagementFactory.getPlatformMBeanServer() + HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) + mxBean.dumpHeap(heapDumpFile, true) + } + + void stop() { + task?.cancel(false) + scheduler.shutdownNow() + } + } } From 0a415ca34c2eaf64d44de09d4b6e4b83f54bdef5 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 14 Aug 2025 17:14:20 -0400 Subject: [PATCH 07/35] Trying to reproduce. --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 0d60304ad91..f6a2442cea1 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 10005, 60005, TimeUnit.MILLISECONDS) + }, 10000, 60005, TimeUnit.MILLISECONDS) } void heapDump(String kind) { diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 7e1edfea332..40ed407cabc 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -140,7 +140,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } writer.write("==============================================\n") } - }, 10003, 60000, TimeUnit.MILLISECONDS) + }, 10000, 60000, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From 7a263b971605926b7ff037117b2f2b052d6cacfa Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 14 Aug 2025 18:45:52 -0400 Subject: [PATCH 08/35] Trying to reproduce. --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index f6a2442cea1..37a97aae974 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 10000, 60005, TimeUnit.MILLISECONDS) + }, 100, 60010, TimeUnit.MILLISECONDS) } void heapDump(String kind) { diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 40ed407cabc..f7881bdb31a 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -140,7 +140,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } writer.write("==============================================\n") } - }, 10000, 60000, TimeUnit.MILLISECONDS) + }, 100, 60000, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From 26d7a120d378d3b58cd447c065b3f515a105ae6d Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 14 Aug 2025 20:25:32 -0400 Subject: [PATCH 09/35] Trying to reproduce. --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 37a97aae974..cbc27893100 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 100, 60010, TimeUnit.MILLISECONDS) + }, 10000, 60010, TimeUnit.MILLISECONDS) } void heapDump(String kind) { diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index f7881bdb31a..40ed407cabc 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -140,7 +140,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } writer.write("==============================================\n") } - }, 100, 60000, TimeUnit.MILLISECONDS) + }, 10000, 60000, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From 91795639aae258fc8161ae91e944a631f7e44e32 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 15 Aug 2025 08:54:39 -0400 Subject: [PATCH 10/35] Trying to reproduce. --- .../java-concurrent/java-concurrent-21/build.gradle | 5 ----- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 5cad74f8ac0..361477f06ff 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -46,11 +46,6 @@ tasks.named("check").configure { dependencies { testImplementation project(':dd-java-agent:instrumentation:trace-annotation') - - // Using Spock 2.4-M6 because it contains fix for deadlock when blocking in mock response generators. - // See: https://github.com/spockframework/spock/pull/1910 - testImplementation libs.spock24.core - testImplementation libs.spock24.junit4 } // Set all compile tasks to use JDK21 but let instrumentation code targets 1.8 compatibility diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index cbc27893100..4a35f80223b 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -224,7 +224,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { } heapDump("test_2") - }, 10000, 60010, TimeUnit.MILLISECONDS) + }, 20000, 60010, TimeUnit.MILLISECONDS) } void heapDump(String kind) { diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 40ed407cabc..8840f71bec9 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -140,7 +140,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } writer.write("==============================================\n") } - }, 10000, 60000, TimeUnit.MILLISECONDS) + }, 20000, 60000, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From 435d111e55ec518217000881cb478328fbe50a94 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 18 Aug 2025 16:56:54 -0400 Subject: [PATCH 11/35] Trying to reproduce. --- .../java-concurrent/java-concurrent-21/build.gradle | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 361477f06ff..acfaefb2188 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -46,6 +46,16 @@ tasks.named("check").configure { dependencies { testImplementation project(':dd-java-agent:instrumentation:trace-annotation') + // Logback supports VirtualThreads since 1.3.12. + // https://logback.qos.ch/news.html#1.3.12 + testImplementation('ch.qos.logback:logback-classic:1.3.15') +} + +configurations.configureEach { + resolutionStrategy { + force 'ch.qos.logback:logback-classic:1.3.15' + force 'ch.qos.logback:logback-core:1.3.15' + } } // Set all compile tasks to use JDK21 but let instrumentation code targets 1.8 compatibility From 92322587c65e231663fa911df28eddbd517d12c9 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 19 Aug 2025 10:56:34 -0400 Subject: [PATCH 12/35] Trying to fix with `useStrictTraceWrites == false` --- .../java-concurrent/java-concurrent-21/build.gradle | 10 ---------- .../groovy/StructuredConcurrencyTest.groovy | 4 ++++ .../src/test/groovy/Lettuce4ClientTestBase.groovy | 5 +++++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index acfaefb2188..361477f06ff 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -46,16 +46,6 @@ tasks.named("check").configure { dependencies { testImplementation project(':dd-java-agent:instrumentation:trace-annotation') - // Logback supports VirtualThreads since 1.3.12. - // https://logback.qos.ch/news.html#1.3.12 - testImplementation('ch.qos.logback:logback-classic:1.3.15') -} - -configurations.configureEach { - resolutionStrategy { - force 'ch.qos.logback:logback-classic:1.3.15' - force 'ch.qos.logback:logback-core:1.3.15' - } } // Set all compile tasks to use JDK21 but let instrumentation code targets 1.8 compatibility diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 4a35f80223b..2713a49f817 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -40,6 +40,10 @@ class StructuredConcurrencyTest extends AgentTestRunner { threadDumpLogger.stop() } + @Override + boolean useStrictTraceWrites() { + return false + } /** * Tests the structured task scope with a single task. diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 8840f71bec9..0c2c582cb9e 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -112,6 +112,11 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { redisServer.stop() } + @Override + boolean useStrictTraceWrites() { + return false + } + // 🔒 Private helper class for thread dump logging private static class ThreadDumpLogger { private final String testName From 6f7706ba62023e4b3532010125e22c546b1b2ef6 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 21 Aug 2025 08:46:50 -0400 Subject: [PATCH 13/35] Trying to reproduce. --- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 5289a330b15..dcf3d9d208f 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -146,7 +146,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } writer.write("==============================================\n") } - }, 20000, 60000, TimeUnit.MILLISECONDS) + }, 20001, 60000, TimeUnit.MILLISECONDS) } void heapDump(String kind) { From b8f1dd3efcd721b89066634353d7488f4144ca3b Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 21 Aug 2025 14:21:10 -0400 Subject: [PATCH 14/35] Rolled back `useStrictTraceWrites == false` --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 6 ------ .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 6 ------ 2 files changed, 12 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 8e15c9f1cbe..90d9336909e 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -40,12 +40,6 @@ class StructuredConcurrencyTest extends AgentTestRunner { threadDumpLogger.stop() } - @Override - boolean useStrictTraceWrites() { - // TODO: Monitor in CI to validate fix effectiveness against freezes. - return false - } - /** * Tests the structured task scope with a single task. */ diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index dcf3d9d208f..c4f8af0a2f2 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -112,12 +112,6 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { redisServer.stop() } - @Override - boolean useStrictTraceWrites() { - // TODO: Monitor in CI to validate fix effectiveness against freezes. - return false - } - // 🔒 Private helper class for thread dump logging private static class ThreadDumpLogger { private final String testName From 4429e5dadea2978f2e62cc84cb337349f3434099 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 21 Aug 2025 18:18:55 -0400 Subject: [PATCH 15/35] Refactored dump logic: start thread in advance and sleep. --- .../groovy/StructuredConcurrencyTest.groovy | 91 ++++++++++--------- .../test/groovy/Lettuce4ClientTestBase.groovy | 86 ++++++++++-------- 2 files changed, 93 insertions(+), 84 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 90d9336909e..a30e2da5c00 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -5,11 +5,7 @@ import datadog.trace.api.Trace import javax.management.MBeanServer import java.lang.management.ManagementFactory import java.util.concurrent.Callable -import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.ScheduledFuture import java.util.concurrent.StructuredTaskScope -import java.util.concurrent.TimeUnit import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace @@ -19,20 +15,10 @@ class StructuredConcurrencyTest extends AgentTestRunner { ThreadDumpLogger threadDumpLogger def setup() { - File reportDir = new File("build") - String fullPath = reportDir.absolutePath.replace("dd-trace-java/dd-java-agent", - "dd-trace-java/workspace/dd-java-agent") - - reportDir = new File(fullPath) - if (!reportDir.exists()) { - println("Folder not found: " + fullPath) - reportDir.mkdirs() - } else println("Folder found: " + fullPath) - // Use the current feature name as the test name String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}" - threadDumpLogger = new ThreadDumpLogger(testName, reportDir) + threadDumpLogger = new ThreadDumpLogger(testName) threadDumpLogger.start() } @@ -51,12 +37,12 @@ class StructuredConcurrencyTest extends AgentTestRunner { when: runUnderTrace("parent") { def task = taskScope.fork(new Callable() { - @Trace(operationName = "child") - @Override - Boolean call() throws Exception { - return true - } - }) + @Trace(operationName = "child") + @Override + Boolean call() throws Exception { + return true + } + }) taskScope.joinUntil(now() + 10) // Wait for 10 seconds at maximum result = task.get() } @@ -195,48 +181,65 @@ class StructuredConcurrencyTest extends AgentTestRunner { } } } + // 🔒 Private helper class for thread dump logging private static class ThreadDumpLogger { private final String testName - private final File outputDir - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor() - private ScheduledFuture task + private Thread task - ThreadDumpLogger(String testName, File outputDir) { + ThreadDumpLogger(String testName) { this.testName = testName - this.outputDir = outputDir } void start() { - task = scheduler.scheduleAtFixedRate({ - heapDump("test_1") - - def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") - try (def writer = new FileWriter(reportFile)) { - writer.write("=== Test: ${testName} ===\n") - writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") - Thread.getAllStackTraces().each { thread, stack -> - writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") - stack.each { writer.write("\tat ${it}\n") } + task = new Thread() { + @Override + void run() { + sleep(10000) + + File outputDir = new File("build") + String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent") + + outputDir = new File(fullPath) + if (!outputDir.exists()) { + println("Folder not found: " + fullPath) + outputDir.mkdirs() + } else println("Folder found: " + fullPath) + + // Use the current feature name as the test name + println("Test name: " + testName) + + heapDump(outputDir, "test_1") + + def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") + + try (def writer = new FileWriter(reportFile)) { + writer.write("=== Test: ${testName} ===\n") + writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") + getAllStackTraces().each { thread, stack -> + writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") + stack.each { writer.write("\tat ${it}\n") } + } + writer.write("==============================================\n") + } + + heapDump(outputDir, "test_2") } - writer.write("==============================================\n") } - - heapDump("test_2") - }, 20000, 60010, TimeUnit.MILLISECONDS) + task.start() } - void heapDump(String kind) { + static void heapDump(File outputDir, String kind) { def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath MBeanServer server = ManagementFactory.getPlatformMBeanServer() HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy( - server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) mxBean.dumpHeap(heapDumpFile, true) } void stop() { - task?.cancel(false) - scheduler.shutdownNow() + task?.interrupt() } } } diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index c4f8af0a2f2..0d72162e111 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -63,28 +63,17 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { embeddedDbUri = "redis://" + dbAddr redisServer = RedisServer.newRedisServer() - // bind to localhost to avoid firewall popup - .setting("bind " + HOST) - // set max memory to avoid problems in CI - .setting("maxmemory 128M") - .port(port).build() + // bind to localhost to avoid firewall popup + .setting("bind " + HOST) + // set max memory to avoid problems in CI + .setting("maxmemory 128M") + .port(port).build() } def setup() { - File reportDir = new File("build") - String fullPath = reportDir.absolutePath.replace("dd-trace-java/dd-java-agent", - "dd-trace-java/workspace/dd-java-agent") - - reportDir = new File(fullPath) - if (!reportDir.exists()) { - println("Folder not found: " + fullPath) - reportDir.mkdirs() - } else println("Folder found: " + fullPath) - // Use the current feature name as the test name String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}" - - threadDumpLogger = new ThreadDumpLogger(testName, reportDir) + threadDumpLogger = new ThreadDumpLogger(testName) threadDumpLogger.start() redisServer.start() @@ -115,45 +104,62 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { // 🔒 Private helper class for thread dump logging private static class ThreadDumpLogger { private final String testName - private final File outputDir - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor() - private ScheduledFuture task + private Thread task - ThreadDumpLogger(String testName, File outputDir) { + ThreadDumpLogger(String testName) { this.testName = testName - this.outputDir = outputDir } void start() { - // new File(outputDir, "${System.currentTimeMillis()}-start-mark.txt") << testName - - task = scheduler.scheduleAtFixedRate({ - heapDump("test") - - def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") - try (def writer = new FileWriter(reportFile)) { - writer.write("=== Test: ${testName} ===\n") - writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") - Thread.getAllStackTraces().each { thread, stack -> - writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") - stack.each { writer.write("\tat ${it}\n") } + task = new Thread() { + @Override + void run() { + println("Dumper started for test: " + testName) + sleep(10000) + + File outputDir = new File("build") + String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent") + + outputDir = new File(fullPath) + if (!outputDir.exists()) { + println("Folder not found: " + fullPath) + outputDir.mkdirs() + } else println("Folder found: " + fullPath) + + // Use the current feature name as the test name + println("Test name: " + testName) + + heapDump(outputDir, "test_1") + + def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") + + try (def writer = new FileWriter(reportFile)) { + writer.write("=== Test: ${testName} ===\n") + writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") + getAllStackTraces().each { thread, stack -> + writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") + stack.each { writer.write("\tat ${it}\n") } + } + writer.write("==============================================\n") + } + + heapDump(outputDir, "test_2") } - writer.write("==============================================\n") } - }, 20001, 60000, TimeUnit.MILLISECONDS) + task.start() } - void heapDump(String kind) { + static void heapDump(File outputDir, String kind) { def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath MBeanServer server = ManagementFactory.getPlatformMBeanServer() HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy( - server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) mxBean.dumpHeap(heapDumpFile, true) } void stop() { - task?.cancel(false) - scheduler.shutdownNow() + task?.interrupt() } } } From ce809d7bfed377fc5a6c7f3490c43c59eacf6e8b Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 21 Aug 2025 19:14:15 -0400 Subject: [PATCH 16/35] Trying to reproduce --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index a30e2da5c00..9f58f92fe61 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -195,7 +195,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { task = new Thread() { @Override void run() { - sleep(10000) + sleep(20000) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 0d72162e111..6b2ff070b7e 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -115,7 +115,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { @Override void run() { println("Dumper started for test: " + testName) - sleep(10000) + sleep(20000) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", From 79204d911f2df8be63693b0c7fc2b521f33d4205 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 21 Aug 2025 20:19:46 -0400 Subject: [PATCH 17/35] Refactored thread dump logic --- .../groovy/StructuredConcurrencyTest.groovy | 19 +++++++++++---- .../test/groovy/Lettuce4ClientTestBase.groovy | 24 +++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 9f58f92fe61..c1487a68420 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -4,6 +4,8 @@ import datadog.trace.api.Trace import javax.management.MBeanServer import java.lang.management.ManagementFactory +import java.lang.management.ThreadInfo +import java.lang.management.ThreadMXBean import java.util.concurrent.Callable import java.util.concurrent.StructuredTaskScope @@ -195,7 +197,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { task = new Thread() { @Override void run() { - sleep(20000) + sleep(12000) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", @@ -217,10 +219,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { try (def writer = new FileWriter(reportFile)) { writer.write("=== Test: ${testName} ===\n") writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") - getAllStackTraces().each { thread, stack -> - writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") - stack.each { writer.write("\tat ${it}\n") } - } + writer.write(threadDump(false, false)) writer.write("==============================================\n") } @@ -238,6 +237,16 @@ class StructuredConcurrencyTest extends AgentTestRunner { mxBean.dumpHeap(heapDumpFile, true) } + private static String threadDump(boolean lockedMonitors, boolean lockedSynchronizers) { + StringBuffer threadDump = new StringBuffer(System.lineSeparator()) + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean() + for(ThreadInfo threadInfo : threadMXBean.dumpAllThreads(lockedMonitors, lockedSynchronizers)) { + threadDump.append(threadInfo.toString()) + } + + return threadDump.toString() + } + void stop() { task?.interrupt() } diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 6b2ff070b7e..b79c525a204 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -11,10 +11,8 @@ import spock.lang.Shared import javax.management.MBeanServer import java.lang.management.ManagementFactory -import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.ScheduledFuture -import java.util.concurrent.TimeUnit +import java.lang.management.ThreadInfo +import java.lang.management.ThreadMXBean import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace @@ -114,8 +112,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { task = new Thread() { @Override void run() { - println("Dumper started for test: " + testName) - sleep(20000) + sleep(12000) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", @@ -137,10 +134,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { try (def writer = new FileWriter(reportFile)) { writer.write("=== Test: ${testName} ===\n") writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") - getAllStackTraces().each { thread, stack -> - writer.write("Thread: ${thread.name}, daemon: ${thread.daemon}\n") - stack.each { writer.write("\tat ${it}\n") } - } + writer.write(threadDump(false, false)) writer.write("==============================================\n") } @@ -158,6 +152,16 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { mxBean.dumpHeap(heapDumpFile, true) } + private static String threadDump(boolean lockedMonitors, boolean lockedSynchronizers) { + StringBuffer threadDump = new StringBuffer(System.lineSeparator()) + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean() + for(ThreadInfo threadInfo : threadMXBean.dumpAllThreads(lockedMonitors, lockedSynchronizers)) { + threadDump.append(threadInfo.toString()) + } + + return threadDump.toString() + } + void stop() { task?.interrupt() } From d777c3c7bbbd12b99c529e7b32f1c48159eb43a9 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 21 Aug 2025 21:03:53 -0400 Subject: [PATCH 18/35] Refactored thread dump logic --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index c1487a68420..16c10e62f6c 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -197,7 +197,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { task = new Thread() { @Override void run() { - sleep(12000) + sleep(11500) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index b79c525a204..298b02fb632 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -112,7 +112,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { task = new Thread() { @Override void run() { - sleep(12000) + sleep(11000) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", From dd2ea8c67ab231a753470f1d0d293f6325158ee0 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 22 Aug 2025 07:59:19 -0400 Subject: [PATCH 19/35] Trying to reproduce --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 2 +- .../lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 16c10e62f6c..c1487a68420 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -197,7 +197,7 @@ class StructuredConcurrencyTest extends AgentTestRunner { task = new Thread() { @Override void run() { - sleep(11500) + sleep(12000) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index 298b02fb632..b79c525a204 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -112,7 +112,7 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { task = new Thread() { @Override void run() { - sleep(11000) + sleep(12000) File outputDir = new File("build") String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", From bb06758e52d0bdebfc28cbd9bab363c9f48fe294 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 22 Aug 2025 16:46:00 -0400 Subject: [PATCH 20/35] Attempt to get dump via gradle --- .../java-concurrent-21/build.gradle | 54 ++++++++++++ .../groovy/StructuredConcurrencyTest.groovy | 86 ------------------- .../instrumentation/lettuce-4/build.gradle | 54 ++++++++++++ .../test/groovy/Lettuce4ClientTestBase.groovy | 83 ------------------ 4 files changed, 108 insertions(+), 169 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 361477f06ff..71485faa244 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -1,3 +1,6 @@ +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + ext { minJavaVersionForTests = JavaVersion.VERSION_21 // Structured concurrency is a preview feature in Java 21. Methods (e.g. ShutdownOnFailure) used in this instrumentation test are no longer available in Java 25, so we set the max version to 24. @@ -56,3 +59,54 @@ compileJava.configure { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 } + +tasks.withType(Test).configureEach { + doFirst { + String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent") + + // outputDir = new File(fullPath) + + println "DEBUG >> ${fullPath}" + + def outDir = new File(fullPath) + outDir.mkdirs() + def outFile = new File(outDir, "${System.currentTimeMillis()}-thread-dump.log") + + // single-thread scheduler (daemon) + def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> + Thread t = new Thread(r, "dump-scheduler") + t.setDaemon(true) + return t + }) + + // schedule the dump job 10s later + def future = scheduler.schedule({ + try { + new ProcessBuilder("jcmd", "0", "Thread.print", "-l") + .redirectErrorStream(true) + .redirectOutput(outFile) + .start().waitFor() + + logger.lifecycle("Thread dump written to: ${outFile.absolutePath}") + } catch (Throwable t) { + logger.warn("Thread dump helper failed: ${t.message}", t) + } finally { + scheduler.shutdown() + } + }, 20, TimeUnit.SECONDS) + + // store handles for cancellation in doLast + ext.dumpFuture = future + ext.dumpScheduler = scheduler + } + + doLast { + // cancel if task completed in <10s + try { + ext.dumpFuture?.cancel(false) + } finally { + ext.dumpScheduler?.shutdownNow() + } + } +} diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index c1487a68420..19b07c9df41 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -2,10 +2,6 @@ import com.sun.management.HotSpotDiagnosticMXBean import datadog.trace.agent.test.AgentTestRunner import datadog.trace.api.Trace -import javax.management.MBeanServer -import java.lang.management.ManagementFactory -import java.lang.management.ThreadInfo -import java.lang.management.ThreadMXBean import java.util.concurrent.Callable import java.util.concurrent.StructuredTaskScope @@ -14,20 +10,6 @@ import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace import static java.time.Instant.now class StructuredConcurrencyTest extends AgentTestRunner { - ThreadDumpLogger threadDumpLogger - - def setup() { - // Use the current feature name as the test name - String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}" - - threadDumpLogger = new ThreadDumpLogger(testName) - threadDumpLogger.start() - } - - def cleanup() { - threadDumpLogger.stop() - } - /** * Tests the structured task scope with a single task. */ @@ -183,72 +165,4 @@ class StructuredConcurrencyTest extends AgentTestRunner { } } } - - // 🔒 Private helper class for thread dump logging - private static class ThreadDumpLogger { - private final String testName - private Thread task - - ThreadDumpLogger(String testName) { - this.testName = testName - } - - void start() { - task = new Thread() { - @Override - void run() { - sleep(12000) - - File outputDir = new File("build") - String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", - "dd-trace-java/workspace/dd-java-agent") - - outputDir = new File(fullPath) - if (!outputDir.exists()) { - println("Folder not found: " + fullPath) - outputDir.mkdirs() - } else println("Folder found: " + fullPath) - - // Use the current feature name as the test name - println("Test name: " + testName) - - heapDump(outputDir, "test_1") - - def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") - - try (def writer = new FileWriter(reportFile)) { - writer.write("=== Test: ${testName} ===\n") - writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") - writer.write(threadDump(false, false)) - writer.write("==============================================\n") - } - - heapDump(outputDir, "test_2") - } - } - task.start() - } - - static void heapDump(File outputDir, String kind) { - def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath - MBeanServer server = ManagementFactory.getPlatformMBeanServer() - HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy( - server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) - mxBean.dumpHeap(heapDumpFile, true) - } - - private static String threadDump(boolean lockedMonitors, boolean lockedSynchronizers) { - StringBuffer threadDump = new StringBuffer(System.lineSeparator()) - ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean() - for(ThreadInfo threadInfo : threadMXBean.dumpAllThreads(lockedMonitors, lockedSynchronizers)) { - threadDump.append(threadInfo.toString()) - } - - return threadDump.toString() - } - - void stop() { - task?.interrupt() - } - } } diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index 3eb173a4343..28e7ebcb938 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -1,3 +1,5 @@ +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit muzzle { pass { @@ -20,3 +22,55 @@ dependencies { latestDepTestImplementation group: 'biz.paluch.redis', name: 'lettuce', version: '4.+' } + + +tasks.withType(Test).configureEach { + doFirst { + String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", + "dd-trace-java/workspace/dd-java-agent") + + // outputDir = new File(fullPath) + + println "DEBUG >> ${fullPath}" + + def outDir = new File(fullPath) + outDir.mkdirs() + def outFile = new File(outDir, "${System.currentTimeMillis()}-thread-dump.log") + + // single-thread scheduler (daemon) + def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> + Thread t = new Thread(r, "dump-scheduler") + t.setDaemon(true) + return t + }) + + // schedule the dump job 10s later + def future = scheduler.schedule({ + try { + new ProcessBuilder("jcmd", "0", "Thread.print", "-l") + .redirectErrorStream(true) + .redirectOutput(outFile) + .start().waitFor() + + logger.lifecycle("Thread dump written to: ${outFile.absolutePath}") + } catch (Throwable t) { + logger.warn("Thread dump helper failed: ${t.message}", t) + } finally { + scheduler.shutdown() + } + }, 20, TimeUnit.SECONDS) + + // store handles for cancellation in doLast + ext.dumpFuture = future + ext.dumpScheduler = scheduler + } + + doLast { + // cancel if task completed in <10s + try { + ext.dumpFuture?.cancel(false) + } finally { + ext.dumpScheduler?.shutdownNow() + } + } +} diff --git a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy index b79c525a204..6d178e33622 100644 --- a/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy +++ b/dd-java-agent/instrumentation/lettuce-4/src/test/groovy/Lettuce4ClientTestBase.groovy @@ -3,17 +3,11 @@ import com.lambdaworks.redis.RedisClient import com.lambdaworks.redis.api.StatefulConnection import com.lambdaworks.redis.api.async.RedisAsyncCommands import com.lambdaworks.redis.api.sync.RedisCommands -import com.sun.management.HotSpotDiagnosticMXBean import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.agent.test.utils.PortUtils import redis.embedded.RedisServer import spock.lang.Shared -import javax.management.MBeanServer -import java.lang.management.ManagementFactory -import java.lang.management.ThreadInfo -import java.lang.management.ThreadMXBean - import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { @@ -38,8 +32,6 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { @Shared RedisServer redisServer - ThreadDumpLogger threadDumpLogger - @Shared Map testHashMap = [ firstname: "John", @@ -69,11 +61,6 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } def setup() { - // Use the current feature name as the test name - String testName = "${specificationContext?.currentSpec?.name ?: "unknown-spec"} : ${specificationContext?.currentFeature?.name ?: "unknown-test"}" - threadDumpLogger = new ThreadDumpLogger(testName) - threadDumpLogger.start() - redisServer.start() redisClient = RedisClient.create(embeddedDbUri) @@ -92,78 +79,8 @@ abstract class Lettuce4ClientTestBase extends VersionedNamingTestBase { } def cleanup() { - threadDumpLogger.stop() - connection.close() redisClient.shutdown() redisServer.stop() } - - // 🔒 Private helper class for thread dump logging - private static class ThreadDumpLogger { - private final String testName - private Thread task - - ThreadDumpLogger(String testName) { - this.testName = testName - } - - void start() { - task = new Thread() { - @Override - void run() { - sleep(12000) - - File outputDir = new File("build") - String fullPath = outputDir.absolutePath.replace("dd-trace-java/dd-java-agent", - "dd-trace-java/workspace/dd-java-agent") - - outputDir = new File(fullPath) - if (!outputDir.exists()) { - println("Folder not found: " + fullPath) - outputDir.mkdirs() - } else println("Folder found: " + fullPath) - - // Use the current feature name as the test name - println("Test name: " + testName) - - heapDump(outputDir, "test_1") - - def reportFile = new File(outputDir, "${System.currentTimeMillis()}-thread-dump.log") - - try (def writer = new FileWriter(reportFile)) { - writer.write("=== Test: ${testName} ===\n") - writer.write("=== Thread Dump Triggered at ${new Date()} ===\n") - writer.write(threadDump(false, false)) - writer.write("==============================================\n") - } - - heapDump(outputDir, "test_2") - } - } - task.start() - } - - static void heapDump(File outputDir, String kind) { - def heapDumpFile = new File(outputDir, "${System.currentTimeMillis()}-heap-dump-${kind}.hprof").absolutePath - MBeanServer server = ManagementFactory.getPlatformMBeanServer() - HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy( - server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class) - mxBean.dumpHeap(heapDumpFile, true) - } - - private static String threadDump(boolean lockedMonitors, boolean lockedSynchronizers) { - StringBuffer threadDump = new StringBuffer(System.lineSeparator()) - ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean() - for(ThreadInfo threadInfo : threadMXBean.dumpAllThreads(lockedMonitors, lockedSynchronizers)) { - threadDump.append(threadInfo.toString()) - } - - return threadDump.toString() - } - - void stop() { - task?.interrupt() - } - } } From 556e780bc9b49304388be686582a55e832239391 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 22 Aug 2025 18:09:58 -0400 Subject: [PATCH 21/35] Fixed codenarc. --- .../src/previewTest/groovy/StructuredConcurrencyTest.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy index 19b07c9df41..0c0280afbaf 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/src/previewTest/groovy/StructuredConcurrencyTest.groovy @@ -1,4 +1,3 @@ -import com.sun.management.HotSpotDiagnosticMXBean import datadog.trace.agent.test.AgentTestRunner import datadog.trace.api.Trace From f55ccb52904d187c9fba5c198cffbc91a0fca122 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 22 Aug 2025 20:12:03 -0400 Subject: [PATCH 22/35] Trying to reproduce. --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 71485faa244..9a1ca704467 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -94,7 +94,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 20, TimeUnit.SECONDS) + }, 60, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future From f2ec31706b1a114fb9228ad8169a297cf6455a87 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 22 Aug 2025 23:18:15 -0400 Subject: [PATCH 23/35] Trying to reproduce. --- .../java-concurrent/java-concurrent-21/build.gradle | 4 +--- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 9a1ca704467..805bfe46ac4 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -65,8 +65,6 @@ tasks.withType(Test).configureEach { String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") - // outputDir = new File(fullPath) - println "DEBUG >> ${fullPath}" def outDir = new File(fullPath) @@ -94,7 +92,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECONDS) + }, 61, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index 28e7ebcb938..ebd270eeac4 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -29,8 +29,6 @@ tasks.withType(Test).configureEach { String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") - // outputDir = new File(fullPath) - println "DEBUG >> ${fullPath}" def outDir = new File(fullPath) From 161991a3b636041ea7b08680130a4d5352daaac7 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 23 Aug 2025 12:29:05 -0400 Subject: [PATCH 24/35] Trying to reproduce with heap dump. --- .../java-concurrent-21/build.gradle | 28 ++++++++++++++---- .../instrumentation/lettuce-4/build.gradle | 29 +++++++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 805bfe46ac4..1bf3415770a 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -64,8 +64,7 @@ tasks.withType(Test).configureEach { doFirst { String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") - - println "DEBUG >> ${fullPath}" + println "DEBUG full path: ${fullPath}" def outDir = new File(fullPath) outDir.mkdirs() @@ -81,18 +80,37 @@ tasks.withType(Test).configureEach { // schedule the dump job 10s later def future = scheduler.schedule({ try { + println "DEBUG threads dump: ${outFile.absolutePath}" + new ProcessBuilder("jcmd", "0", "Thread.print", "-l") .redirectErrorStream(true) .redirectOutput(outFile) .start().waitFor() - logger.lifecycle("Thread dump written to: ${outFile.absolutePath}") + println "DEBUG heap dumps" + def listOutput = "jcmd -l".execute().text.readLines() + listOutput.each { line -> + def pid = line.substring(0, line.indexOf(' ')) + println "DEBUG PID: ${pid}" + + def dumpFileName = "${fullPath}/${System.currentTimeMillis()}-${pid}.hprof" + + println "DEBUG dump file name: ${dumpFileName}" + + def cmd = "jcmd ${pid} GC.heap_dump ${dumpFileName}" + + println "DEBUG jcmd: ${cmd}" + + def res = cmd.execute().waitFor() + + println "DEBUG dump res: ${res}" + } } catch (Throwable t) { - logger.warn("Thread dump helper failed: ${t.message}", t) + logger.warn("Dumping failed: ${t.message}", t) } finally { scheduler.shutdown() } - }, 61, TimeUnit.SECONDS) + }, 60, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index ebd270eeac4..ec84ba6c76d 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -23,13 +23,11 @@ dependencies { latestDepTestImplementation group: 'biz.paluch.redis', name: 'lettuce', version: '4.+' } - tasks.withType(Test).configureEach { doFirst { String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") - - println "DEBUG >> ${fullPath}" + println "DEBUG full path: ${fullPath}" def outDir = new File(fullPath) outDir.mkdirs() @@ -45,18 +43,37 @@ tasks.withType(Test).configureEach { // schedule the dump job 10s later def future = scheduler.schedule({ try { + println "DEBUG get thread dumps: ${outFile.absolutePath}" + new ProcessBuilder("jcmd", "0", "Thread.print", "-l") .redirectErrorStream(true) .redirectOutput(outFile) .start().waitFor() - logger.lifecycle("Thread dump written to: ${outFile.absolutePath}") + println "DEBUG get heap dumps" + def listOutput = "jcmd -l".execute().text.readLines() + listOutput.each { line -> + def pid = line.substring(0, line.indexOf(' ')) + println "DEBUG PID: ${pid}" + + def dumpFileName = "${fullPath}/${System.currentTimeMillis()}-${pid}.hprof" + + println "DEBUG dump file name: ${dumpFileName}" + + def cmd = "jcmd ${pid} GC.heap_dump ${dumpFileName}" + + println "DEBUG jcmd: ${cmd}" + + def res = cmd.execute().waitFor() + + println "DEBUG dump res: ${res}" + } } catch (Throwable t) { - logger.warn("Thread dump helper failed: ${t.message}", t) + logger.warn("Dumping failed: ${t.message}", t) } finally { scheduler.shutdown() } - }, 20, TimeUnit.SECONDS) + }, 60, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future From cd1c49c2dea2de75e16b3fa192b2a0de4bab46e5 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 23 Aug 2025 15:17:20 -0400 Subject: [PATCH 25/35] Skip non-gradle heap dumps --- .../java-concurrent/java-concurrent-21/build.gradle | 7 ++++++- dd-java-agent/instrumentation/lettuce-4/build.gradle | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 1bf3415770a..19e8722ba4d 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -90,8 +90,13 @@ tasks.withType(Test).configureEach { println "DEBUG heap dumps" def listOutput = "jcmd -l".execute().text.readLines() listOutput.each { line -> + println "DEBUG PID: ${line}" + + if (!line.contains("gradle")) { + return + } + def pid = line.substring(0, line.indexOf(' ')) - println "DEBUG PID: ${pid}" def dumpFileName = "${fullPath}/${System.currentTimeMillis()}-${pid}.hprof" diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index ec84ba6c76d..df109b88f6e 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -43,18 +43,23 @@ tasks.withType(Test).configureEach { // schedule the dump job 10s later def future = scheduler.schedule({ try { - println "DEBUG get thread dumps: ${outFile.absolutePath}" + println "DEBUG threads dump: ${outFile.absolutePath}" new ProcessBuilder("jcmd", "0", "Thread.print", "-l") .redirectErrorStream(true) .redirectOutput(outFile) .start().waitFor() - println "DEBUG get heap dumps" + println "DEBUG heap dumps" def listOutput = "jcmd -l".execute().text.readLines() listOutput.each { line -> + println "DEBUG PID: ${line}" + + if (!line.contains("gradle")) { + return + } + def pid = line.substring(0, line.indexOf(' ')) - println "DEBUG PID: ${pid}" def dumpFileName = "${fullPath}/${System.currentTimeMillis()}-${pid}.hprof" From 331e308ca97c6ef547fcbee0c7aeabb711959abc Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 23 Aug 2025 19:00:10 -0400 Subject: [PATCH 26/35] Trying to reproduce. --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 19e8722ba4d..5489d20d951 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -115,7 +115,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECONDS) + }, 61, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index df109b88f6e..a499e43b4cf 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -78,7 +78,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECONDS) + }, 61, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future From a6270998c520f4d1b69401f60ee1823f0d34599c Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 25 Aug 2025 08:10:53 -0400 Subject: [PATCH 27/35] Trying to reproduce. --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 5489d20d951..a6a8869a4d9 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -115,7 +115,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 61, TimeUnit.SECONDS) + }, 62, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index a499e43b4cf..309b2a2c687 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -78,7 +78,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 61, TimeUnit.SECONDS) + }, 62, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future From 255e66cc08171d41c4baae7b6b0a86b531148b86 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 25 Aug 2025 11:13:16 -0400 Subject: [PATCH 28/35] Trying to reproduce. --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index a6a8869a4d9..19e8722ba4d 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -115,7 +115,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 62, TimeUnit.SECONDS) + }, 60, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index 309b2a2c687..df109b88f6e 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -78,7 +78,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 62, TimeUnit.SECONDS) + }, 60, TimeUnit.SECONDS) // store handles for cancellation in doLast ext.dumpFuture = future From 5ebc09ceacbb3e17b29580d099f490ec59fa1bb8 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 25 Aug 2025 13:17:54 -0400 Subject: [PATCH 29/35] Limit Heap dumps only for Gradle Test Executors. --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 19e8722ba4d..b56d1b01de5 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -92,7 +92,7 @@ tasks.withType(Test).configureEach { listOutput.each { line -> println "DEBUG PID: ${line}" - if (!line.contains("gradle")) { + if (!line.contains("Gradle Test Executor")) { return } diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index df109b88f6e..71bc892d857 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -55,7 +55,7 @@ tasks.withType(Test).configureEach { listOutput.each { line -> println "DEBUG PID: ${line}" - if (!line.contains("gradle")) { + if (!line.contains("Gradle Test Executor")) { return } From 714fdb9e4559c740524e07c3f1d803683b7baeb3 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 27 Aug 2025 12:37:58 -0400 Subject: [PATCH 30/35] Minor refactoring. --- .gitlab/collect_reports.sh | 2 +- .../java-concurrent-21/build.gradle | 28 ++++++------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/.gitlab/collect_reports.sh b/.gitlab/collect_reports.sh index 1b30beb6a6e..6b16d7da472 100755 --- a/.gitlab/collect_reports.sh +++ b/.gitlab/collect_reports.sh @@ -62,7 +62,7 @@ function process_reports () { cp -r workspace/$project_to_save/build/reports/* $report_path/ 2>/dev/null || true cp workspace/$project_to_save/build/hs_err_pid*.log $report_path/ 2>/dev/null || true cp workspace/$project_to_save/build/javacore*.txt $report_path/ 2>/dev/null || true - cp workspace/$project_to_save/build/*.* $report_path/ 2>/dev/null || true + cp workspace/$project_to_save/build/dumps/*.* $report_path/ 2>/dev/null || true fi } diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index b56d1b01de5..81e118ee9a5 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -62,13 +62,11 @@ compileJava.configure { tasks.withType(Test).configureEach { doFirst { - String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", + String buildDir = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") - println "DEBUG full path: ${fullPath}" - def outDir = new File(fullPath) - outDir.mkdirs() - def outFile = new File(outDir, "${System.currentTimeMillis()}-thread-dump.log") + def dupmsDir = new File(buildDir, "dumps") + dupmsDir.mkdirs() // single-thread scheduler (daemon) def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> @@ -80,14 +78,13 @@ tasks.withType(Test).configureEach { // schedule the dump job 10s later def future = scheduler.schedule({ try { - println "DEBUG threads dump: ${outFile.absolutePath}" + def threadDumpFile = new File(dupmsDir, "${System.currentTimeMillis()}-thread-dump.log") new ProcessBuilder("jcmd", "0", "Thread.print", "-l") .redirectErrorStream(true) - .redirectOutput(outFile) + .redirectOutput(threadDumpFile) .start().waitFor() - println "DEBUG heap dumps" def listOutput = "jcmd -l".execute().text.readLines() listOutput.each { line -> println "DEBUG PID: ${line}" @@ -97,18 +94,9 @@ tasks.withType(Test).configureEach { } def pid = line.substring(0, line.indexOf(' ')) - - def dumpFileName = "${fullPath}/${System.currentTimeMillis()}-${pid}.hprof" - - println "DEBUG dump file name: ${dumpFileName}" - - def cmd = "jcmd ${pid} GC.heap_dump ${dumpFileName}" - - println "DEBUG jcmd: ${cmd}" - - def res = cmd.execute().waitFor() - - println "DEBUG dump res: ${res}" + def heapDumpFile = "${dupmsDir.absolutePath}/${System.currentTimeMillis()}-${pid}.hprof" + def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}" + cmd.execute().waitFor() } } catch (Throwable t) { logger.warn("Dumping failed: ${t.message}", t) From f382692421dc1ecf0621279ed94328d28c9060e5 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 27 Aug 2025 13:58:45 -0400 Subject: [PATCH 31/35] Minor refactoring. --- .../instrumentation/lettuce-4/build.gradle | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index 71bc892d857..1fa1c9e1830 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -25,13 +25,11 @@ dependencies { tasks.withType(Test).configureEach { doFirst { - String fullPath = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", + String buildDir = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") - println "DEBUG full path: ${fullPath}" - def outDir = new File(fullPath) - outDir.mkdirs() - def outFile = new File(outDir, "${System.currentTimeMillis()}-thread-dump.log") + def dupmsDir = new File(buildDir, "dumps") + dupmsDir.mkdirs() // single-thread scheduler (daemon) def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> @@ -43,14 +41,13 @@ tasks.withType(Test).configureEach { // schedule the dump job 10s later def future = scheduler.schedule({ try { - println "DEBUG threads dump: ${outFile.absolutePath}" + def threadDumpFile = new File(dupmsDir, "${System.currentTimeMillis()}-thread-dump.log") new ProcessBuilder("jcmd", "0", "Thread.print", "-l") .redirectErrorStream(true) - .redirectOutput(outFile) + .redirectOutput(threadDumpFile) .start().waitFor() - println "DEBUG heap dumps" def listOutput = "jcmd -l".execute().text.readLines() listOutput.each { line -> println "DEBUG PID: ${line}" @@ -60,18 +57,9 @@ tasks.withType(Test).configureEach { } def pid = line.substring(0, line.indexOf(' ')) - - def dumpFileName = "${fullPath}/${System.currentTimeMillis()}-${pid}.hprof" - - println "DEBUG dump file name: ${dumpFileName}" - - def cmd = "jcmd ${pid} GC.heap_dump ${dumpFileName}" - - println "DEBUG jcmd: ${cmd}" - - def res = cmd.execute().waitFor() - - println "DEBUG dump res: ${res}" + def heapDumpFile = "${dupmsDir.absolutePath}/${System.currentTimeMillis()}-${pid}.hprof" + def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}" + cmd.execute().waitFor() } } catch (Throwable t) { logger.warn("Dumping failed: ${t.message}", t) From be9dae212e6660a9b35e2f884b1eac66743bd99c Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 27 Aug 2025 16:01:59 -0400 Subject: [PATCH 32/35] Minor refactoring. --- .../java-concurrent-21/build.gradle | 24 ++++++++++--------- .../instrumentation/lettuce-4/build.gradle | 24 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 81e118ee9a5..41156c7637a 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -62,20 +62,22 @@ compileJava.configure { tasks.withType(Test).configureEach { doFirst { + // This manipulation is requred to move files into folder + // that will be collected by `collect_reports.sh` script. String buildDir = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") def dupmsDir = new File(buildDir, "dumps") dupmsDir.mkdirs() - // single-thread scheduler (daemon) + // Scheduler to execute logic that will dump threads and heap def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> Thread t = new Thread(r, "dump-scheduler") t.setDaemon(true) return t }) - // schedule the dump job 10s later + // Schedule the dump job: def future = scheduler.schedule({ try { def threadDumpFile = new File(dupmsDir, "${System.currentTimeMillis()}-thread-dump.log") @@ -85,15 +87,15 @@ tasks.withType(Test).configureEach { .redirectOutput(threadDumpFile) .start().waitFor() - def listOutput = "jcmd -l".execute().text.readLines() - listOutput.each { line -> - println "DEBUG PID: ${line}" - - if (!line.contains("Gradle Test Executor")) { + // Collect PIDs of all Java processes: + def jvmProcesses = "jcmd -l".execute().text.readLines() + jvmProcesses.each { p -> + // Process only test executors. + if (!p.contains("Gradle Test Executor")) { return } - def pid = line.substring(0, line.indexOf(' ')) + def pid = p.substring(0, p.indexOf(' ')) def heapDumpFile = "${dupmsDir.absolutePath}/${System.currentTimeMillis()}-${pid}.hprof" def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}" cmd.execute().waitFor() @@ -103,15 +105,15 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECONDS) + }, 60, TimeUnit.SECCONDS) // You can adjust this line with required delay. - // store handles for cancellation in doLast + // Store handles for cancellation in doLast: ext.dumpFuture = future ext.dumpScheduler = scheduler } doLast { - // cancel if task completed in <10s + // Cancel if task completed faster than specified timeout. try { ext.dumpFuture?.cancel(false) } finally { diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index 1fa1c9e1830..8034aa22328 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -25,20 +25,22 @@ dependencies { tasks.withType(Test).configureEach { doFirst { + // This manipulation is requred to move files into folder + // that will be collected by `collect_reports.sh` script. String buildDir = layout.buildDirectory.asFile.get().absolutePath.replace("dd-trace-java/dd-java-agent", "dd-trace-java/workspace/dd-java-agent") def dupmsDir = new File(buildDir, "dumps") dupmsDir.mkdirs() - // single-thread scheduler (daemon) + // Scheduler to execute logic that will dump threads and heap def scheduler = Executors.newSingleThreadScheduledExecutor({ r -> Thread t = new Thread(r, "dump-scheduler") t.setDaemon(true) return t }) - // schedule the dump job 10s later + // Schedule the dump job: def future = scheduler.schedule({ try { def threadDumpFile = new File(dupmsDir, "${System.currentTimeMillis()}-thread-dump.log") @@ -48,15 +50,15 @@ tasks.withType(Test).configureEach { .redirectOutput(threadDumpFile) .start().waitFor() - def listOutput = "jcmd -l".execute().text.readLines() - listOutput.each { line -> - println "DEBUG PID: ${line}" - - if (!line.contains("Gradle Test Executor")) { + // Collect PIDs of all Java processes: + def jvmProcesses = "jcmd -l".execute().text.readLines() + jvmProcesses.each { p -> + // Process only test executors. + if (!p.contains("Gradle Test Executor")) { return } - def pid = line.substring(0, line.indexOf(' ')) + def pid = p.substring(0, p.indexOf(' ')) def heapDumpFile = "${dupmsDir.absolutePath}/${System.currentTimeMillis()}-${pid}.hprof" def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}" cmd.execute().waitFor() @@ -66,15 +68,15 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECONDS) + }, 60, TimeUnit.SECCONDS) // You can adjust this line with required delay. - // store handles for cancellation in doLast + // Store handles for cancellation in doLast: ext.dumpFuture = future ext.dumpScheduler = scheduler } doLast { - // cancel if task completed in <10s + // Cancel if task completed faster than specified timeout. try { ext.dumpFuture?.cancel(false) } finally { From d920e9225fa6f8e60c52bde5cc66c976fcec406c Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 27 Aug 2025 16:46:16 -0400 Subject: [PATCH 33/35] Fixed typo --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 41156c7637a..423507561ee 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -105,7 +105,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECCONDS) // You can adjust this line with required delay. + }, 60, TimeUnit.SECONDS) // You can adjust this line with required delay. // Store handles for cancellation in doLast: ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index 8034aa22328..d7990427a98 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -68,7 +68,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECCONDS) // You can adjust this line with required delay. + }, 60, TimeUnit.SECONDS) // You can adjust this line with required delay. // Store handles for cancellation in doLast: ext.dumpFuture = future From 70f5de031c4403dd0242d869f29251ce6d57497e Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 Aug 2025 14:16:45 -0400 Subject: [PATCH 34/35] Trying to reproduce freeze on CI. --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index 5375ed236b5..fc4db3ca23b 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -107,7 +107,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECONDS) // You can adjust this line with required delay. + }, 62, TimeUnit.SECONDS) // You can adjust this line with required delay. // Store handles for cancellation in doLast: ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index d7990427a98..b74398018a6 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -68,7 +68,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 60, TimeUnit.SECONDS) // You can adjust this line with required delay. + }, 62, TimeUnit.SECONDS) // You can adjust this line with required delay. // Store handles for cancellation in doLast: ext.dumpFuture = future From fdfbadd9c88154e913a55edc520bdc00ffdaaaf0 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Sat, 30 Aug 2025 20:22:22 -0400 Subject: [PATCH 35/35] Trying to reproduce freeze on CI. --- .../java-concurrent/java-concurrent-21/build.gradle | 2 +- dd-java-agent/instrumentation/lettuce-4/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle index fc4db3ca23b..ca129da40fd 100644 --- a/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/java-concurrent-21/build.gradle @@ -107,7 +107,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 62, TimeUnit.SECONDS) // You can adjust this line with required delay. + }, 65, TimeUnit.SECONDS) // You can adjust this line with required delay. // Store handles for cancellation in doLast: ext.dumpFuture = future diff --git a/dd-java-agent/instrumentation/lettuce-4/build.gradle b/dd-java-agent/instrumentation/lettuce-4/build.gradle index b74398018a6..f818f129968 100644 --- a/dd-java-agent/instrumentation/lettuce-4/build.gradle +++ b/dd-java-agent/instrumentation/lettuce-4/build.gradle @@ -68,7 +68,7 @@ tasks.withType(Test).configureEach { } finally { scheduler.shutdown() } - }, 62, TimeUnit.SECONDS) // You can adjust this line with required delay. + }, 65, TimeUnit.SECONDS) // You can adjust this line with required delay. // Store handles for cancellation in doLast: ext.dumpFuture = future