8361725: Do not load Java agent with "-Xshare:dump -XX:+AOTClassLinking"

Reviewed-by: matsaave, ccheung

Author: iklam

src/hotspot/share/cds/cdsConfig.cpp

@@ -598,6 +598,7 @@ void CDSConfig::check_aotmode_create() {
   //
   // Since application is not executed in the assembly phase, there's no need to load
   // the agents anyway -- no one will notice that the agents are not loaded.
+  log_info(aot)("Disabled all JVMTI agents during -XX:AOTMode=create");
   JvmtiAgentList::disable_agent_list();
 }
 
@@ -702,6 +703,13 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla
     }
   }
 
+  if (is_dumping_classic_static_archive() && AOTClassLinking) {
+    if (JvmtiAgentList::disable_agent_list()) {
+      FLAG_SET_ERGO(AllowArchivingWithJavaAgent, false);
+      log_warning(cds)("Disabled all JVMTI agents with -Xshare:dump -XX:+AOTClassLinking");
+    }
+  }
+
   return true;
 }
 

src/hotspot/share/prims/jvmtiAgentList.cpp

@@ -273,11 +273,13 @@ JvmtiAgent* JvmtiAgentList::lookup(JvmtiEnv* env, void* f_ptr) {
   return nullptr;
 }
 
-void JvmtiAgentList::disable_agent_list() {
+bool JvmtiAgentList::disable_agent_list() {
 #if INCLUDE_CDS
-  assert(CDSConfig::is_dumping_final_static_archive(), "use this only for -XX:AOTMode=create!");
   assert(!Universe::is_bootstrapping() && !Universe::is_fully_initialized(), "must do this very early");
-  log_info(aot)("Disabled all JVMTI agents during -XX:AOTMode=create");
-  _head = nullptr; // Pretend that no agents have been added.
+  if (_head != nullptr) {
+    _head = nullptr; // Pretend that no agents have been added.
+    return true;
+  }
 #endif
+  return false;
 }

src/hotspot/share/prims/jvmtiAgentList.hpp

@@ -83,7 +83,7 @@ class JvmtiAgentList : AllStatic {
   static Iterator java_agents();
   static Iterator native_agents();
   static Iterator xrun_agents();
-  static void disable_agent_list() NOT_JVMTI_RETURN;
+  static bool disable_agent_list() NOT_JVMTI_RETURN_(false);
 };
 
 #endif // SHARE_PRIMS_JVMTIAGENTLIST_HPP

test/hotspot/jtreg/runtime/cds/appcds/aotCache/JavaAgent.java

@@ -24,7 +24,18 @@
 
 
 /*
- * @test
+ * @test id=static
+ * @bug 8361725
+ * @summary -javaagent should be disabled with -Xshare:dump -XX:+AOTClassLinking
+ * @requires vm.cds.supports.aot.class.linking
+ * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds/test-classes
+ * @build JavaAgent JavaAgentTransformer Util
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar app.jar JavaAgentApp JavaAgentApp$ShouldBeTransformed
+ * @run driver JavaAgent STATIC
+ */
+
+/*
+ * @test id=aot
  * @summary -javaagent should be allowed in AOT workflow. However, classes transformed/redefined by agents will not
  *          be cached.
  * @requires vm.cds.supports.aot.class.linking
@@ -68,7 +79,13 @@ public String classpath(RunMode runMode) {
 
         @Override
         public String[] vmArgs(RunMode runMode) {
-            return new String[] { "-javaagent:" + agentJar, "-Xlog:aot,cds"};
+            return new String[] {
+                "-XX:+UnlockDiagnosticVMOptions",
+                "-XX:+AllowArchivingWithJavaAgent",
+                "-javaagent:" + agentJar,
+                "-Xlog:aot,cds",
+                "-XX:+AOTClassLinking",
+            };
         }
 
         @Override
@@ -80,7 +97,18 @@ public String[] appCommandLine(RunMode runMode) {
 
         @Override
         public void checkExecution(OutputAnalyzer out, RunMode runMode) throws Exception {
-            String agentLoadedMsg = "JavaAgentTransformer.premain() is called";
+            if (isAOTWorkflow()) {
+                checkExecutionForAOTWorkflow(out, runMode);
+            } else {
+                checkExecutionForStaticWorkflow(out, runMode);
+            }
+        }
+
+        static String agentLoadedMsg = "JavaAgentTransformer.premain() is called";
+        static String agentPremainFinished = "JavaAgentTransformer::premain() is finished";
+
+        public void checkExecutionForAOTWorkflow(OutputAnalyzer out, RunMode runMode) throws Exception {
+
             if (runMode.isApplicationExecuted()) {
                 out.shouldContain(agentLoadedMsg);
                 out.shouldContain("Transforming: JavaAgentApp$ShouldBeTransformed; Class<?> = null");
@@ -91,12 +119,26 @@ public void checkExecution(OutputAnalyzer out, RunMode runMode) throws Exception
 
             switch (runMode) {
             case RunMode.TRAINING:
+                out.shouldContain(agentPremainFinished);
                 out.shouldContain("Skipping JavaAgentApp$ShouldBeTransformed: From ClassFileLoadHook");
                 out.shouldContain("Skipping JavaAgentTransformer: Unsupported location");
                 break;
             case RunMode.ASSEMBLY:
                 out.shouldContain("Disabled all JVMTI agents during -XX:AOTMode=create");
+                out.shouldNotContain(agentPremainFinished);
+                break;
+            }
+
+        }
+
+        public void checkExecutionForStaticWorkflow(OutputAnalyzer out, RunMode runMode) throws Exception {
+            switch (runMode) {
+            case RunMode.DUMP_STATIC:
+                out.shouldContain("Disabled all JVMTI agents with -Xshare:dump -XX:+AOTClassLinking");
+                out.shouldNotContain(agentPremainFinished);
                 break;
+            default:
+                out.shouldContain(agentPremainFinished);
             }
         }
     }

test/hotspot/jtreg/runtime/cds/appcds/aotCache/JavaAgentTransformer.java

@@ -22,6 +22,7 @@
  *
  */
 
+import java.lang.System.Logger.Level;
 import java.lang.instrument.ClassFileTransformer;
 import java.lang.instrument.IllegalClassFormatException;
 import java.lang.instrument.Instrumentation;
@@ -30,11 +31,14 @@
 // This class is available on the classpath so it can be accessed by JavaAgentApp
 public class JavaAgentTransformer  implements ClassFileTransformer {
     private static Instrumentation savedInstrumentation;
+    private static final System.Logger LOGGER = System.getLogger(JavaAgentTransformer.class.getName());
 
     public static void premain(String agentArguments, Instrumentation instrumentation) {
         System.out.println("JavaAgentTransformer.premain() is called");
         instrumentation.addTransformer(new JavaAgentTransformer(), /*canRetransform=*/true);
         savedInstrumentation = instrumentation;
+
+        LOGGER.log(Level.WARNING, "JavaAgentTransformer::premain() is finished");
     }
 
     public static Instrumentation getInstrumentation() {