Add experimental Build Tools API (BTAPI) for compilation (bazelbuild#1325)

erikkerber · restingbull · oliviernotteghem · commit 24e529dd3428 · 2025-06-11T23:03:19.000-07:00
* Add experimental Build Tools API (BTAPI) for compilation

* Provision build tools API with http_archive over Maven

* Fixup maven_install

* Add build tools api to the integration tests

---------

Co-authored-by: Corbin McNeely-Smith &lt;restingbull@mcneely-smith.com&gt;
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -19,6 +19,7 @@ use_repo(
     "com_github_jetbrains_kotlin",
     "com_github_jetbrains_kotlin_git",
     "com_github_pinterest_ktlint",
+    "kotlin_build_tools_impl",
     "kotlinx_serialization_core_jvm",
     "kotlinx_serialization_json",
     "kotlinx_serialization_json_jvm",
diff --git a/MODULE.release.bazel b/MODULE.release.bazel
@@ -24,6 +24,7 @@ use_repo(
     "kotlinx_serialization_core_jvm",
     "kotlinx_serialization_json",
     "kotlinx_serialization_json_jvm",
+    "kotlin_build_tools_impl",
 )
 
 register_toolchains("//kotlin/internal:default_toolchain")
diff --git a/docs/kotlin.md b/docs/kotlin.md
@@ -505,8 +505,10 @@ This allows setting options and dependencies independently from the initial plug
 load("@rules_kotlin//kotlin:core.bzl", "define_kt_toolchain")
 
 define_kt_toolchain(<a href="#define_kt_toolchain-name">name</a>, <a href="#define_kt_toolchain-language_version">language_version</a>, <a href="#define_kt_toolchain-api_version">api_version</a>, <a href="#define_kt_toolchain-jvm_target">jvm_target</a>, <a href="#define_kt_toolchain-experimental_use_abi_jars">experimental_use_abi_jars</a>,
-                    <a href="#define_kt_toolchain-experimental_strict_kotlin_deps">experimental_strict_kotlin_deps</a>, <a href="#define_kt_toolchain-experimental_report_unused_deps">experimental_report_unused_deps</a>,
-                    <a href="#define_kt_toolchain-experimental_reduce_classpath_mode">experimental_reduce_classpath_mode</a>, <a href="#define_kt_toolchain-experimental_multiplex_workers">experimental_multiplex_workers</a>, <a href="#define_kt_toolchain-javac_options">javac_options</a>,
+                    <a href="#define_kt_toolchain-experimental_treat_internal_as_private_in_abi_jars">experimental_treat_internal_as_private_in_abi_jars</a>,
+                    <a href="#define_kt_toolchain-experimental_remove_private_classes_in_abi_jars">experimental_remove_private_classes_in_abi_jars</a>, <a href="#define_kt_toolchain-experimental_strict_kotlin_deps">experimental_strict_kotlin_deps</a>,
+                    <a href="#define_kt_toolchain-experimental_report_unused_deps">experimental_report_unused_deps</a>, <a href="#define_kt_toolchain-experimental_reduce_classpath_mode">experimental_reduce_classpath_mode</a>,
+                    <a href="#define_kt_toolchain-experimental_multiplex_workers">experimental_multiplex_workers</a>, <a href="#define_kt_toolchain-experimental_build_tools_api">experimental_build_tools_api</a>, <a href="#define_kt_toolchain-javac_options">javac_options</a>,
                     <a href="#define_kt_toolchain-kotlinc_options">kotlinc_options</a>, <a href="#define_kt_toolchain-jvm_stdlibs">jvm_stdlibs</a>, <a href="#define_kt_toolchain-jvm_runtime">jvm_runtime</a>, <a href="#define_kt_toolchain-jacocorunner">jacocorunner</a>, <a href="#define_kt_toolchain-exec_compatible_with">exec_compatible_with</a>,
                     <a href="#define_kt_toolchain-target_compatible_with">target_compatible_with</a>, <a href="#define_kt_toolchain-target_settings">target_settings</a>)
 </pre>
@@ -527,6 +529,7 @@ Define the Kotlin toolchain.
 | <a id="define_kt_toolchain-experimental_report_unused_deps"></a>experimental_report_unused_deps |  <p align="center"> - </p>   |  `None` |
 | <a id="define_kt_toolchain-experimental_reduce_classpath_mode"></a>experimental_reduce_classpath_mode |  <p align="center"> - </p>   |  `None` |
 | <a id="define_kt_toolchain-experimental_multiplex_workers"></a>experimental_multiplex_workers |  <p align="center"> - </p>   |  `None` |
+| <a id="define_kt_toolchain-experimental_build_tools_api"></a>experimental_build_tools_api |  <p align="center"> - </p>   |  `None` |
 | <a id="define_kt_toolchain-javac_options"></a>javac_options |  <p align="center"> - </p>   |  `Label("@rules_kotlin//kotlin/internal:default_javac_options")` |
 | <a id="define_kt_toolchain-kotlinc_options"></a>kotlinc_options |  <p align="center"> - </p>   |  `Label("@rules_kotlin//kotlin/internal:default_kotlinc_options")` |
 | <a id="define_kt_toolchain-jvm_stdlibs"></a>jvm_stdlibs |  <p align="center"> - </p>   |  `None` |
diff --git a/kotlin/internal/jvm/compile.bzl b/kotlin/internal/jvm/compile.bzl
@@ -539,6 +539,7 @@ def _run_kt_builder_action(
     args.add("--strict_kotlin_deps", toolchains.kt.experimental_strict_kotlin_deps)
     args.add_all("--classpath", compile_deps.compile_jars)
     args.add("--reduced_classpath_mode", toolchains.kt.experimental_reduce_classpath_mode)
+    args.add("--build_tools_api", toolchains.kt.experimental_build_tools_api)
     args.add_all("--sources", srcs.all_srcs, omit_if_empty = True)
     args.add_all("--source_jars", srcs.src_jars + generated_src_jars, omit_if_empty = True)
     args.add_all("--deps_artifacts", deps_artifacts, omit_if_empty = True)
diff --git a/kotlin/internal/toolchains.bzl b/kotlin/internal/toolchains.bzl
@@ -92,6 +92,7 @@ def _kotlin_toolchain_impl(ctx):
         experimental_compile_with_transitive_deps = ctx.attr.experimental_compile_with_transitive_deps,
         experimental_track_class_usage = ctx.attr.experimental_track_class_usage,
         experimental_track_resource_usage = ctx.attr.experimental_track_resource_usage,
+        experimental_build_tools_api = ctx.attr.experimental_build_tools_api,
         javac_options = ctx.attr.javac_options[JavacOptions] if ctx.attr.javac_options else None,
         kotlinc_options = ctx.attr.kotlinc_options[KotlincOptions] if ctx.attr.kotlinc_options else None,
         empty_jar = ctx.file._empty_jar,
@@ -258,6 +259,10 @@ _kt_toolchain = rule(
             doc = "Compile with or without transitive dependencies from the classpath",
             default = True,
         ),
+        "experimental_build_tools_api": attr.bool(
+            doc = "Enables experimental support for Build Tools API integration",
+            default = False,
+        ),
         "javac_options": attr.label(
             doc = "Compiler options for javac",
             providers = [JavacOptions],
@@ -347,6 +352,7 @@ def define_kt_toolchain(
         experimental_kover_exclude = [],
         experimental_kover_exclude_annotation = [],
         experimental_kover_exclude_inherited_from = [],
+        experimental_build_tools_api = None,
         javac_options = Label("//kotlin/internal:default_javac_options"),
         kotlinc_options = Label("//kotlin/internal:default_kotlinc_options"),
         jvm_stdlibs = None,
@@ -381,6 +387,7 @@ def define_kt_toolchain(
         experimental_kover_exclude = experimental_kover_exclude,
         experimental_kover_exclude_annotation = experimental_kover_exclude_annotation,
         experimental_kover_exclude_inherited_from = experimental_kover_exclude_inherited_from,
+        experimental_build_tools_api = experimental_build_tools_api,
         javac_options = javac_options,
         kotlinc_options = kotlinc_options,
         visibility = ["//visibility:public"],
diff --git a/kotlin/settings/BUILD.bazel b/kotlin/settings/BUILD.bazel
@@ -36,3 +36,9 @@ bool_flag(
     build_setting_default = False,
     visibility = ["//visibility:public"],
 )
+
+bool_flag(
+    name = "experimental_build_tools_api",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
diff --git a/kotlin/settings/BUILD.release.bazel b/kotlin/settings/BUILD.release.bazel
@@ -27,3 +27,10 @@ bool_flag(
     build_setting_default = False,
     visibility = ["//visibility:public"],
 )
+
+# --@rules_kotlin//kotlin/settings:experimental_build_tools_api=True
+bool_flag(
+    name = "experimental_build_tools_api",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
diff --git a/src/main/kotlin/BUILD.release.bazel b/src/main/kotlin/BUILD.release.bazel
@@ -47,6 +47,7 @@ java_binary(
         "//kotlin/compiler:symbol-processing-cmdline",
         "//src/main/kotlin/io/bazel/kotlin/compiler",
         "@com_github_jetbrains_kotlin//:home",
+        "@kotlin_build_tools_impl//jar",
         "@kotlinx_serialization_core_jvm//jar",
         "@kotlinx_serialization_json//jar",
         "@kotlinx_serialization_json_jvm//jar",
@@ -55,6 +56,7 @@ java_binary(
         "-D@com_github_jetbrains_kotlinx...serialization-core-jvm=$(rlocationpath @kotlinx_serialization_core_jvm//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json=$(rlocationpath @kotlinx_serialization_json//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json-jvm=$(rlocationpath @kotlinx_serialization_json_jvm//jar)",
+        "-D@com_github_jetbrains_kotlin...build-tools-impl=$(rlocationpath @kotlin_build_tools_impl//jar)",
         "-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rlocationpath //kotlin/compiler:jvm-abi-gen)",
         "-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rlocationpath //kotlin/compiler:kotlin-compiler)",
         "-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath //kotlin/compiler:kotlin-annotation-processing)",
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/cmd/BUILD.bazel b/src/main/kotlin/io/bazel/kotlin/builder/cmd/BUILD.bazel
@@ -25,6 +25,7 @@ kt_bootstrap_binary(
         "//src/main/kotlin:skip-code-gen",
         "//src/main/kotlin/io/bazel/kotlin/compiler:compiler.jar",
         "@com_github_jetbrains_kotlin//:home",
+        "@kotlin_build_tools_impl//jar",
         "@kotlinx_serialization_core_jvm//jar",
         "@kotlinx_serialization_json//jar",
         "@kotlinx_serialization_json_jvm//jar",
@@ -33,6 +34,7 @@ kt_bootstrap_binary(
         "-D@com_github_jetbrains_kotlinx...serialization-core-jvm=$(rlocationpath @kotlinx_serialization_core_jvm//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json=$(rlocationpath @kotlinx_serialization_json//jar)",
         "-D@com_github_jetbrains_kotlinx...serialization-json-jvm=$(rlocationpath @kotlinx_serialization_json_jvm//jar)",
+        "-D@com_github_jetbrains_kotlin...build-tools-impl=$(rlocationpath @kotlin_build_tools_impl//jar)",
         "-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rlocationpath //kotlin/compiler:jvm-abi-gen)",
         "-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rlocationpath //kotlin/compiler:kotlin-compiler)",
         "-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath //kotlin/compiler:kotlin-annotation-processing)",
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt b/src/main/kotlin/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt
@@ -89,6 +89,7 @@ class KotlinBuilder
         KSP_OPTS("--ksp_opts"),
         TRACK_CLASS_USAGE("--track_class_usage"),
         TRACK_RESOURCE_USAGE("--track_resource_usage"),
+        BUILD_TOOLS_API("--build_tools_api"),
       }
     }
 
@@ -168,6 +169,9 @@ class KotlinBuilder
         reducedClasspathMode = argMap.mandatorySingle(KotlinBuilderFlags.REDUCED_CLASSPATH_MODE)
         trackClassUsage = argMap.mandatorySingle(KotlinBuilderFlags.TRACK_CLASS_USAGE)
         trackResourceUsage = argMap.mandatorySingle(KotlinBuilderFlags.TRACK_RESOURCE_USAGE)
+        argMap.optionalSingle(KotlinBuilderFlags.BUILD_TOOLS_API)?.let {
+          buildToolsApi = it == "true"
+        }
         this
       }
 
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt b/src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt
@@ -33,7 +33,7 @@ const val X_FRIENDS_PATH_SEPARATOR = ","
 class KotlinJvmTaskExecutor
   @Inject
   internal constructor(
-    private val compiler: KotlinToolchain.KotlincInvoker,
+    private val compilerBuilder: KotlinToolchain.KotlincInvokerBuilder,
     private val plugins: InternalCompilerPlugins,
   ) {
     private fun combine(
@@ -54,6 +54,8 @@ class KotlinJvmTaskExecutor
       context: CompilationTaskContext,
       task: JvmCompilationTask,
     ) {
+      val compiler = compilerBuilder.build(context.info.buildToolsApi)
+
       val preprocessedTask =
         task
           .preProcessingSteps(context)
diff --git a/src/main/kotlin/io/bazel/kotlin/builder/toolchain/KotlinToolchain.kt b/src/main/kotlin/io/bazel/kotlin/builder/toolchain/KotlinToolchain.kt
@@ -128,6 +128,13 @@ class KotlinToolchain private constructor(
         ).toPath()
     }
 
+    private val BUILD_TOOLS_API by lazy {
+      BazelRunFiles
+        .resolveVerifiedFromProperty(
+          "@com_github_jetbrains_kotlin...build-tools-impl",
+        ).toPath()
+    }
+
     private val JAVA_HOME by lazy {
       FileSystems
         .getDefault()
@@ -147,6 +154,7 @@ class KotlinToolchain private constructor(
         JAVA_HOME,
         KOTLINC.verified().absoluteFile,
         COMPILER.verified().absoluteFile,
+        BUILD_TOOLS_API.verified().absoluteFile,
         JVM_ABI_PLUGIN.verified().absoluteFile,
         SKIP_CODE_GEN_PLUGIN.verified().absoluteFile,
         JDEPS_GEN_PLUGIN.verified().absoluteFile,
@@ -162,6 +170,7 @@ class KotlinToolchain private constructor(
     fun createToolchain(
       javaHome: Path,
       kotlinc: File,
+      buildTools: File,
       compiler: File,
       jvmAbiGenFile: File,
       skipCodeGenFile: File,
@@ -177,6 +186,7 @@ class KotlinToolchain private constructor(
         listOf(
           kotlinc,
           compiler,
+          buildTools,
           // plugins *must* be preloaded. Not doing so causes class conflicts
           // (and a NoClassDef err) in the compiler extension interfaces.
           // This may cause issues in accepting user defined compiler plugins.
@@ -270,7 +280,7 @@ class KotlinToolchain private constructor(
     val id: String,
   )
 
-  open class KotlinCliToolInvoker internal constructor(
+  open class KotlincInvoker internal constructor(
     toolchain: KotlinToolchain,
     clazz: String,
   ) {
@@ -303,12 +313,19 @@ class KotlinToolchain private constructor(
   }
 
   @Singleton
-  class KotlincInvoker
+  class KotlincInvokerBuilder
     @Inject
     constructor(
-      toolchain: KotlinToolchain,
-    ) : KotlinCliToolInvoker(
-        toolchain.toolchainWithReflect(),
-        "io.bazel.kotlin.compiler.BazelK2JVMCompiler",
-      )
+      private val toolchain: KotlinToolchain,
+    ) {
+      fun build(useExperimentalBuildToolsAPI: Boolean): KotlincInvoker {
+        val clazz =
+          if (useExperimentalBuildToolsAPI) {
+            "io.bazel.kotlin.compiler.BuildToolsAPICompiler"
+          } else {
+            "io.bazel.kotlin.compiler.BazelK2JVMCompiler"
+          }
+        return KotlincInvoker(toolchain = toolchain, clazz = clazz)
+      }
+    }
 }
diff --git a/src/main/kotlin/io/bazel/kotlin/compiler/BuildToolsAPICompiler.kt b/src/main/kotlin/io/bazel/kotlin/compiler/BuildToolsAPICompiler.kt
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2025 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.bazel.kotlin.compiler
+
+import org.jetbrains.kotlin.buildtools.api.CompilationResult
+import org.jetbrains.kotlin.buildtools.api.CompilationService
+import org.jetbrains.kotlin.buildtools.api.ExperimentalBuildToolsApi
+import org.jetbrains.kotlin.buildtools.api.ProjectId
+import org.jetbrains.kotlin.cli.common.ExitCode
+import java.util.UUID
+
+@Suppress("unused")
+class BuildToolsAPICompiler {
+  @OptIn(ExperimentalBuildToolsApi::class)
+  fun exec(
+    errStream: java.io.PrintStream,
+    vararg args: String,
+  ): ExitCode {
+    System.setProperty("zip.handler.uses.crc.instead.of.timestamp", "true")
+
+    val kotlinService = CompilationService.loadImplementation(this.javaClass.classLoader!!)
+    // The execution configuration. Controls in-process vs daemon execution strategies. Default is in-process.
+    val executionConfig = kotlinService.makeCompilerExecutionStrategyConfiguration()
+    // The compilation configuration. Controls everything related to incremental compilation.
+    val compilationConfig =
+      kotlinService.makeJvmCompilationConfiguration().apply {
+        // useLogger(BasicKotlinLogger(true, "/tmp/kotlin_log/$label.log"))
+      }
+
+    val result =
+      kotlinService.compileJvm(
+        ProjectId.ProjectUUID(UUID.randomUUID()),
+        executionConfig,
+        compilationConfig,
+        emptyList(),
+        args.toList(),
+      )
+
+    // BTAPI returns a different type than K2JVMCompiler (CompilationResult vs ExitCode).
+    return when (result) {
+      CompilationResult.COMPILATION_SUCCESS -> ExitCode.OK
+      CompilationResult.COMPILATION_ERROR -> ExitCode.COMPILATION_ERROR
+      CompilationResult.COMPILATION_OOM_ERROR -> ExitCode.OOM_ERROR
+      CompilationResult.COMPILER_INTERNAL_ERROR -> ExitCode.INTERNAL_ERROR
+    }
+  }
+}
diff --git a/src/main/kotlin/io/bazel/kotlin/test/BazelIntegrationTestRunner.kt b/src/main/kotlin/io/bazel/kotlin/test/BazelIntegrationTestRunner.kt
diff --git a/src/main/protobuf/kotlin_model.proto b/src/main/protobuf/kotlin_model.proto
diff --git a/src/main/starlark/core/repositories/initialize.release.bzl b/src/main/starlark/core/repositories/initialize.release.bzl
diff --git a/src/main/starlark/core/repositories/versions.bzl b/src/main/starlark/core/repositories/versions.bzl
diff --git a/src/test/kotlin/io/bazel/kotlin/builder/KotlinAbstractTestBuilder.java b/src/test/kotlin/io/bazel/kotlin/builder/KotlinAbstractTestBuilder.java
diff --git a/src/test/kotlin/io/bazel/kotlin/defs.bzl b/src/test/kotlin/io/bazel/kotlin/defs.bzl

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ use_repo(`
`24`	`24`	`"kotlinx_serialization_core_jvm",`
`25`	`25`	`"kotlinx_serialization_json",`
`26`	`26`	`"kotlinx_serialization_json_jvm",`
	`27`	`+ "kotlin_build_tools_impl",`
`27`	`28`	`)`
`28`	`29`
`29`	`30`	`register_toolchains("//kotlin/internal:default_toolchain")`
Original file line number	Diff line number	Diff line change
`@@ -89,6 +89,7 @@ class KotlinBuilder`
`89`	`89`	`KSP_OPTS("--ksp_opts"),`
`90`	`90`	`TRACK_CLASS_USAGE("--track_class_usage"),`
`91`	`91`	`TRACK_RESOURCE_USAGE("--track_resource_usage"),`
	`92`	`+ BUILD_TOOLS_API("--build_tools_api"),`
`92`	`93`	`}`
`93`	`94`	`}`
`94`	`95`
`@@ -168,6 +169,9 @@ class KotlinBuilder`
`168`	`169`	`reducedClasspathMode = argMap.mandatorySingle(KotlinBuilderFlags.REDUCED_CLASSPATH_MODE)`
`169`	`170`	`trackClassUsage = argMap.mandatorySingle(KotlinBuilderFlags.TRACK_CLASS_USAGE)`
`170`	`171`	`trackResourceUsage = argMap.mandatorySingle(KotlinBuilderFlags.TRACK_RESOURCE_USAGE)`
	`172`	`+ argMap.optionalSingle(KotlinBuilderFlags.BUILD_TOOLS_API)?.let {`
	`173`	`+ buildToolsApi = it == "true"`
	`174`	`+ }`
`171`	`175`	`this`
`172`	`176`	`}`
`173`	`177`