8359827: Test runtime/Thread/ThreadCountLimit.java need loop increasing the limit

test/hotspot/jtreg/runtime/Thread/ThreadCountLimit.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -73,14 +73,27 @@ public static void main(String[] args) throws Exception {
       if (Platform.isLinux()) {
         // On Linux this test sometimes hits the limit for the maximum number of memory mappings,
         // which leads to various other failure modes. Run this test with a limit on how many
-        // threads the process is allowed to create, so we hit that limit first.
-
-        final String ULIMIT_CMD = "ulimit -u 4096";
+        // threads the process is allowed to create, so we hit that limit first. What we want is
+        // for another "limit" processes to be available, but ulimit doesn't work that way and
+        // if there are already many running processes we could fail to even start the JVM properly.
+        // So we loop increasing the limit until we get a successful run. This is not foolproof.
+        int pLimit = 4096;
+        final String ULIMIT_CMD = "ulimit -u ";
         ProcessBuilder pb = ProcessTools.createTestJavaProcessBuilder(ThreadCountLimit.class.getName());
         String javaCmd = ProcessTools.getCommandLine(pb);
-        // Relaunch the test with args.length > 0, and the ulimit set
-        ProcessTools.executeCommand("bash", "-c", ULIMIT_CMD + " && " + javaCmd + " dummy")
-                    .shouldHaveExitValue(0);
+        for (int i = 1; i <= 10; i++) {
+            // Relaunch the test with args.length > 0, and the ulimit set
+            String cmd = ULIMIT_CMD + Integer.toString(pLimit * i) + " && " + javaCmd + " dummy";
+            System.out.println("Trying: bash -c " + cmd);
+            OutputAnalyzer oa = ProcessTools.executeCommand("bash", "-c", cmd);
+            int exitValue = oa.getExitValue();
+            switch (exitValue) {
+              case 0: System.out.println("Success!"); return;
+              case 1: System.out.println("Retry ..."); continue;
+              default: oa.shouldHaveExitValue(0); // generate error report
+            }
+        }
+        throw new Error("Failed to perform a successful run!");
       } else {
         // Not Linux so run directly.
         test();