Set cstack=vm for tests and improve stackwalk test coverage

ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java

@@ -4,10 +4,12 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -107,8 +109,42 @@ protected static Map<String, Object> mapOf(Object ... vals) {
     return map;
   }
 
+  protected static class TestConfiguration {
+      String os;        // OS (Linux, musl, MacOSX or Windows)
+      String arch;      // x86 or aarch64
+      String jvm;       // hotspot, J9 or Graal
+      String cstack;    // vm, vmx, fp, dwarf
+
+      protected TestConfiguration(String os, String arch, String jvm, String cstack) {
+          this.os =os;
+          this.arch =arch;
+          this.jvm =jvm;
+          this.cstack =cstack;
+      }
+
+      public boolean match(String os, String arch, String jvm, String cstack) {
+          return (os.equals(this.os) || "*".equals(this.os)) &&
+                  (arch.equals(this.arch) || "*".equals(this.arch)) &&
+                  (jvm.equals(this.jvm) || "*".equals(this.jvm)) &&
+                  (cstack.equals(this.cstack) || "*".equals(this.cstack));
+      }
+  };
+
+  private List<TestConfiguration> blacklist = new ArrayList<>();
+
   protected AbstractProfilerTest(Map<String, Object> testParams) {
     this.testParams = testParams != null ? new HashMap<>(testParams) : Collections.emptyMap();
+
+    /* Black list:
+     * - J9 does not support cstack=vm or vmx
+     *
+     */
+    blacklist.add(new TestConfiguration("*", "*", "J9", "vm"));
+    blacklist.add(new TestConfiguration("*", "*", "J9", "vmx"));
+    appendBlackList(blacklist);
+  }
+
+  protected void appendBlackList(List<TestConfiguration> blackList) {
   }
 
   protected AbstractProfilerTest() {
@@ -174,7 +210,7 @@ public void setupProfiler(TestInfo testInfo) throws Exception {
     Path rootDir = Paths.get("/tmp/recordings");
     Files.createDirectories(rootDir);
 
-    String cstack = (String)testParams.get("cstack");
+    String cstack = getCStack();
 
     if (cstack != null) {
       rootDir = rootDir.resolve(cstack);
@@ -192,6 +228,10 @@ public void setupProfiler(TestInfo testInfo) throws Exception {
     before();
   }
 
+  protected String getCStack() {
+      return (String) testParams.get("cstack");
+  }
+
   @AfterEach
   public void cleanup() throws Exception {
     after();
@@ -275,13 +315,60 @@ public final void registerCurrentThreadForWallClockProfiling() {
     profiler.addThread();
   }
 
+  protected final String defaultCStack() {
+      return Platform.isJ9() ? "fp" : "vm";
+  }
+
+  private static String getOS() {
+      if (Platform.isWindows()) {
+          return "win";
+      } else if (Platform.isMac()) {
+          return "mac";
+      } else if (Platform.isMusl()) {
+          return "musl";
+      } else {
+          return "linux";
+      }
+  }
+
+  private static String getArch() {
+      if (Platform.isAarch64()) {
+          return "aarch64";
+      } else {
+          return "x86";
+      }
+  }
+
+  private static String getJVM() {
+      if (Platform.isJ9()) {
+          return "J9";
+      } else if (Platform.isGraal()) {
+          return "graal";
+      } else {
+          return "hostspot";
+      }
+  }
+
+  protected boolean isOnBlackList(String cstack) {
+      String os = getOS();
+      String arch = getArch();
+      String jvm = getJVM();
+
+      for (TestConfiguration c: blacklist) {
+          if (c.match(os, arch, jvm, cstack)) {
+              return true;
+          }
+      }
+      return false;
+  }
+
   private String getAmendedProfilerCommand() {
     String profilerCommand = getProfilerCommand();
-    String testCstack = (String)testParams.get("cstack");
+    String testCstack = getCStack();
     if (testCstack != null) {
       profilerCommand += ",cstack=" + testCstack;
     } else if(!(ALLOW_NATIVE_CSTACKS || profilerCommand.contains("cstack="))) {
-      profilerCommand += ",cstack=fp";
+      profilerCommand += ",cstack=" + defaultCStack();
     }
     // FIXME - test framework doesn't seem to be forking each test, so need to sync
     //  these across test cases for now
@@ -337,7 +424,7 @@ protected IItemCollection verifyEvents(Path recording, String eventType, boolean
   }
 
   protected final void verifyCStackSettings() {
-    String cstack = (String)testParams.get("cstack");
+    String cstack = (String)getCStack();
     if (cstack == null) {
       // not a forced cstack mode
       return;

ddprof-test/src/test/java/com/datadoghq/profiler/classgc/ClassGCTest.java

@@ -24,7 +24,7 @@
 public class ClassGCTest extends AbstractProfilerTest {
     @Override
     protected String getProfilerCommand() {
-        return "cpu=1ms,wall=1ms,filter=0,memory=524288:L,cstack=fp";
+        return "cpu=1ms,wall=1ms,filter=0,memory=524288:L";
     }
 
     private static final String CLASS_NAME = "code.Worker";

ddprof-test/src/test/java/com/datadoghq/profiler/jfr/CpuDumpSmokeTest.java

@@ -1,19 +1,35 @@
 package com.datadoghq.profiler.jfr;
 
-import org.junit.jupiter.api.Test;
+import com.datadoghq.profiler.junit.CStack;
+import com.datadoghq.profiler.junit.RetryTest;
+
+import java.util.concurrent.ExecutionException;
+
 import org.junit.jupiter.api.Timeout;
-import org.junitpioneer.jupiter.RetryingTest;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.params.provider.ValueSource;
 
 public class CpuDumpSmokeTest extends JfrDumpTest {
+    public CpuDumpSmokeTest(@CStack String cstack) {
+        super(cstack);
+    }
 
     @Override
     protected String getProfilerCommand() {
-        return "cpu=1ms,cstack=fp";
+        return "cpu=1ms";
     }
 
-    @RetryingTest(3)
     @Timeout(value = 60)
-    public void test() throws Exception {
+    @RetryTest(3)
+    @TestTemplate
+    @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"})
+    public void test(@CStack String cstack) throws ExecutionException, InterruptedException, Exception {
+        if (!isOnBlackList(cstack)) {
+            test();
+        }
+    }
+
+    private void test() throws Exception {
         runTest("datadog.ExecutionSample");
     }
 }

ddprof-test/src/test/java/com/datadoghq/profiler/jfr/JfrDumpTest.java

@@ -1,6 +1,8 @@
 package com.datadoghq.profiler.jfr;
 
-import com.datadoghq.profiler.AbstractProfilerTest;
+import com.datadoghq.profiler.CStackAwareAbstractProfilerTest;
+import com.datadoghq.profiler.junit.RetryTest;
+import com.datadoghq.profiler.junit.CStack;
 import com.datadoghq.profiler.Platform;
 
 import java.io.File;
@@ -9,7 +11,10 @@
 
 import org.junit.jupiter.api.Assumptions;
 
-public abstract class JfrDumpTest extends AbstractProfilerTest {
+public abstract class JfrDumpTest extends CStackAwareAbstractProfilerTest {
+    public JfrDumpTest(@CStack String cstack) {
+        super(cstack);
+    }
 
     public void runTest(String eventName) throws Exception {
         runTest(eventName, "method1", "method2", "method3");

ddprof-test/src/test/java/com/datadoghq/profiler/jfr/ObjectSampleDumpSmokeTest.java

@@ -1,13 +1,20 @@
 package com.datadoghq.profiler.jfr;
 
 import com.datadoghq.profiler.Platform;
+import com.datadoghq.profiler.junit.CStack;
+import com.datadoghq.profiler.junit.RetryTest;
 
-import org.junit.jupiter.api.Assumptions;
-import org.junit.jupiter.api.Timeout;
+import java.util.concurrent.ExecutionException;
 
-import org.junitpioneer.jupiter.RetryingTest;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.params.provider.ValueSource;
 
 public class ObjectSampleDumpSmokeTest extends JfrDumpTest {
+    public ObjectSampleDumpSmokeTest(@CStack String cstack) {
+        super(cstack);
+    }
+
     @Override
     protected boolean isPlatformSupported() {
         return !Platform.isJavaVersion(8) && !Platform.isJ9();
@@ -18,9 +25,17 @@ protected String getProfilerCommand() {
         return "memory=32:a";
     }
 
-    @RetryingTest(5)
+    @RetryTest(5)
+    @TestTemplate
+    @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"})
     @Timeout(value = 300)
-    public void test() throws Exception {
+    public void test(@CStack String cstack) throws ExecutionException, InterruptedException, Exception {
+        if (!isOnBlackList(cstack)) {
+            test();
+        }
+    }
+
+    private void test() throws Exception {
         runTest("datadog.ObjectSample", 3, "method3");
     }
 }

ddprof-test/src/test/java/com/datadoghq/profiler/jfr/WallclockDumpSmokeTest.java

@@ -1,19 +1,35 @@
 package com.datadoghq.profiler.jfr;
 
-import org.junit.jupiter.api.Timeout;
+import com.datadoghq.profiler.junit.CStack;
+import com.datadoghq.profiler.junit.RetryTest;
+
+import java.util.concurrent.ExecutionException;
 
-import org.junitpioneer.jupiter.RetryingTest;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.params.provider.ValueSource;
 
 public class WallclockDumpSmokeTest extends JfrDumpTest {
+    public WallclockDumpSmokeTest(@CStack String cstack) {
+        super(cstack);
+    }
 
     @Override
     protected String getProfilerCommand() {
         return "wall=5ms";
     }
 
-    @RetryingTest(3)
+    @RetryTest(3)
     @Timeout(value = 60)
-    public void test() throws Exception {
+    @TestTemplate
+    @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"})
+    public void test(@CStack String cstack) throws ExecutionException, InterruptedException, Exception {
+        if (!isOnBlackList(cstack)) {
+            test();
+        }
+    }
+
+    private void test() throws Exception {
         registerCurrentThreadForWallClockProfiling();
         runTest("datadog.MethodSample");
     }

ddprof-test/src/test/java/com/datadoghq/profiler/memleak/GCGenerationsTest.java

@@ -20,7 +20,7 @@
 public class GCGenerationsTest extends AbstractProfilerTest {
     @Override
     protected String getProfilerCommand() {
-        return "memory=128,generations=true,cstack=fp";
+        return "memory=128,generations=true";
     }
 
     @Override

ddprof-test/src/test/java/com/datadoghq/profiler/memleak/MemleakProfilerTest.java

@@ -20,7 +20,7 @@
 public class MemleakProfilerTest extends AbstractProfilerTest {
     @Override
     protected String getProfilerCommand() {
-        return "memory=524288:L:0.5,cstack=fp";
+        return "memory=524288:L:0.5";
     }
 
     @Override

ddprof-test/src/test/java/com/datadoghq/profiler/nativelibs/NativeLibrariesTest.java

@@ -1,12 +1,18 @@
 package com.datadoghq.profiler.nativelibs;
 
-import com.datadoghq.profiler.AbstractProfilerTest;
+import com.datadoghq.profiler.CStackAwareAbstractProfilerTest;
+import com.datadoghq.profiler.junit.RetryTest;
+import com.datadoghq.profiler.junit.CStack;
 import com.datadoghq.profiler.Platform;
+
 import com.github.luben.zstd.Zstd;
 import net.jpountz.lz4.LZ4Compressor;
 import net.jpountz.lz4.LZ4Factory;
 import net.jpountz.lz4.LZ4FastDecompressor;
 import net.jpountz.lz4.LZ4SafeDecompressor;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.params.provider.ValueSource;
+
 import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.Test;
 import org.junitpioneer.jupiter.RetryingTest;
@@ -21,6 +27,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
@@ -35,14 +42,26 @@
  * we should be able to parse the dwarf info section of these common libraries on linux without segfaulting
  * and we should be able to unwind the native stacks
  * */
-public class NativeLibrariesTest extends AbstractProfilerTest {
+public class NativeLibrariesTest extends CStackAwareAbstractProfilerTest {
     @Override
     protected String getProfilerCommand() {
-        return "cpu=1ms,cstack=" + (Platform.isMac() ? "fp" : "dwarf");
+        return "cpu=1ms";
+    }
+
+    public NativeLibrariesTest(@CStack String cstack) {
+        super(cstack);
+    }
+
+    @RetryTest(5)
+    @TestTemplate
+    @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"})
+    public void test(@CStack String cstack) throws ExecutionException, InterruptedException, Exception {
+        if (!isOnBlackList(cstack)) {
+            test();
+        }
     }
 
-    @RetryingTest(3)
-    public void test() {
+    private void test() {
         String config = System.getProperty("ddprof_test.config");
         boolean isSanitizer = config.endsWith("san");
 
@@ -95,7 +114,9 @@ public void test() {
         assertTrue(libraryCounters.containsKey("LZ4"), "no lz4-java samples");
         // snappy is problematic on musl; we are not running it
         // for some reason it is not also appearing in sanitized runs
-        assertTrue(isMusl || isSanitizer || libraryCounters.containsKey("SNAPPY"), "no snappy-java samples");
+        // TODO: Missing "SNAPPY" frame with cstack=vm
+        assertTrue(isMusl || isSanitizer || libraryCounters.containsKey("SNAPPY") || "vm".equals(getCStack()),
+                "no snappy-java samples");
         assertTrue(libraryCounters.containsKey("ZSTD"), "no zstd-jni samples");
         modeCounters.forEach((mode, count) -> System.err.println(mode + ": " + count.get()));
         libraryCounters.forEach((lib, count) -> System.err.println(lib + ": " + count.get()));