From 838af47c0e2c6aa338c67fdabcbccc2b6872867c Mon Sep 17 00:00:00 2001 From: Adam Semenenko <152864218+adam-enko@users.noreply.github.com> Date: Thu, 12 Jun 2025 18:49:07 +0200 Subject: [PATCH 1/2] generate CDS for Dokka workers --- .../src/main/kotlin/internal/CdsSource.kt | 178 ++++++++++++++++++ .../main/kotlin/tasks/DokkaGenerateTask.kt | 16 ++ 2 files changed, 194 insertions(+) create mode 100644 dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt diff --git a/dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt new file mode 100644 index 0000000000..c8b60db66d --- /dev/null +++ b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt @@ -0,0 +1,178 @@ +/* + * Copyright 2014-2025 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package org.jetbrains.dokka.gradle.internal + +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.provider.ValueSource +import org.gradle.api.provider.ValueSourceParameters +import org.gradle.process.ExecOperations +import org.jetbrains.kotlin.konan.file.use +import java.io.File +import java.io.OutputStream +import java.io.RandomAccessFile +import java.math.BigInteger +import java.nio.channels.FileChannel +import java.nio.channels.FileLock +import java.nio.channels.OverlappingFileLockException +import java.nio.file.Files +import java.security.DigestOutputStream +import java.security.MessageDigest +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock +import java.util.jar.JarFile +import javax.inject.Inject +import kotlin.concurrent.withLock + + +internal abstract class CdsSource +@Inject +internal constructor( + private val execOps: ExecOperations +) : ValueSource { + + interface Parameters : ValueSourceParameters { + val classpath: ConfigurableFileCollection + } + + private val classpathChecksum: String by lazy { + checksum(parameters.classpath) + } + + private val cacheDir: File by lazy { + val osName = System.getProperty("os.name").lowercase() + val homeDir = System.getProperty("user.home") + val appDataDir = System.getenv("APP_DATA") ?: homeDir + + val userCacheDir = when { + "win" in osName -> "$appDataDir/Caches/" + "mac" in osName -> "$homeDir/Library/Caches/" + "nix" in osName -> "$homeDir/.cache/" + else -> "$homeDir/.cache/" + } + + File(userCacheDir).resolve("dokka").apply { + mkdirs() + } + } + + private val cdsFile: File by lazy { + cacheDir.resolve("$classpathChecksum.jsa") + } + private val lockFile: File by lazy { + cacheDir.resolve("$classpathChecksum.lock") + } + + override fun obtain(): File { + lock.withLock { + RandomAccessFile(lockFile, "rw").use { + it.channel.lockWithRetries().use { + if (!cdsFile.exists()) { + generateStaticCds() + } + println("Using CDS ${cdsFile.absoluteFile.invariantSeparatorsPath}") + return cdsFile + } + } + } + } + + private fun generateStaticCds() { + + val classListFile = Files.createTempFile("asd", "classlist").toFile() + parameters.classpath.files.flatMap { file -> + getClassNamesFromJarFile(file) + } + .toSet() + .joinToString("\n") + .let { + classListFile.writeText(it) + } + + execOps.javaexec { + jvmArgs( + "-Xshare:dump", + "-XX:SharedArchiveFile=${cdsFile.absoluteFile.invariantSeparatorsPath}", + "-XX:SharedClassListFile=${classListFile.absoluteFile.invariantSeparatorsPath}" + ) + classpath(parameters.classpath) + } + } + + companion object { + private val lock: Lock = ReentrantLock() + } +} + + +private fun checksum( + files: ConfigurableFileCollection +): String { + val md = MessageDigest.getInstance("md5") + DigestOutputStream(nullOutputStream(), md).use { os -> + os.write(files.asPath.encodeToByteArray()) + } + return BigInteger(1, md.digest()).toString(16) + .padStart(md.digestLength * 2, '0') +} + +private fun checksum( + files: Collection +): String { + val md = MessageDigest.getInstance("md5") + DigestOutputStream(nullOutputStream(), md).use { os -> + files.forEach { file -> + file.inputStream().use { it.copyTo(os) } + } + } + return BigInteger(1, md.digest()).toString(16) + .padStart(md.digestLength * 2, '0') +} + +private fun nullOutputStream(): OutputStream = + object : OutputStream() { + override fun write(b: Int) {} + } + + +private fun getClassNamesFromJarFile(source: File): Set { + JarFile(source).use { jarFile -> + return jarFile.entries().asSequence() + .filter { it.name.endsWith(".class") } + .map { entry -> + entry.name + .replace("/", ".") + .removeSuffix(".class") + } + .toSet() + } +} + +private fun FileChannel.lockWithRetries(): FileLock { + var retries = 0 + while (true) { + try { + return lock() + } + /* + Catching the OverlappingFileLockException which is caused by the same jvm (process) already having locked the file. + Since we do use a static re-entrant lock as a monitor to the cache, this can only happen + when this code is running in the same JVM but with in complete isolation + (e.g. Gradle classpath isolation, or composite builds). + + If we detect this case, we retry the locking after a short period, constantly logging that we're blocked + by some other thread using the cache. + + The risk of deadlocking here is low, since we can only get into this code path, *if* + the code is very isolated and somebody locked the file. + */ + catch (t: OverlappingFileLockException) { + Thread.sleep(25) + retries++ +// if (retries % 10 == 0) { +//// logInfo("Waiting to acquire lock: $file") +// } + } + } +} diff --git a/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt index 5f02fa25a8..8e6ad0fbcb 100644 --- a/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt +++ b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt @@ -10,8 +10,10 @@ import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.RegularFileProperty import org.gradle.api.model.ObjectFactory import org.gradle.api.provider.Property +import org.gradle.api.provider.ProviderFactory import org.gradle.api.tasks.* import org.gradle.kotlin.dsl.newInstance +import org.gradle.kotlin.dsl.of import org.gradle.kotlin.dsl.submit import org.gradle.workers.WorkerExecutor import org.jetbrains.dokka.DokkaConfiguration @@ -19,6 +21,7 @@ import org.jetbrains.dokka.DokkaConfigurationImpl import org.jetbrains.dokka.gradle.DokkaBasePlugin.Companion.jsonMapper import org.jetbrains.dokka.gradle.engine.parameters.DokkaGeneratorParametersSpec import org.jetbrains.dokka.gradle.engine.parameters.builders.DokkaParametersBuilder +import org.jetbrains.dokka.gradle.internal.CdsSource import org.jetbrains.dokka.gradle.internal.DokkaPluginParametersContainer import org.jetbrains.dokka.gradle.internal.InternalDokkaGradlePluginApi import org.jetbrains.dokka.gradle.workers.ClassLoaderIsolation @@ -50,6 +53,10 @@ constructor( pluginsConfiguration: DokkaPluginParametersContainer, ) : DokkaBaseTask() { + @InternalDokkaGradlePluginApi + @get:Inject + protected open val providers: ProviderFactory get() = error("injected") + private val dokkaParametersBuilder = DokkaParametersBuilder(archives) /** @@ -160,6 +167,15 @@ constructor( isolation.minHeapSize.orNull?.let(this::setMinHeapSize) isolation.jvmArgs.orNull?.filter { it.isNotBlank() }?.let(this::setJvmArgs) isolation.systemProperties.orNull?.let(this::systemProperties) + + val cds = providers.of(CdsSource::class) { + parameters { + classpath.from(runtimeClasspath) + } + } + jvmArgs( + "-XX:SharedArchiveFile=${cds.get().absoluteFile.invariantSeparatorsPath}" + ) } } } From bd2a2880e9dc9defb344db46fabfcd1eec5eba98 Mon Sep 17 00:00:00 2001 From: Adam Semenenko <152864218+adam-enko@users.noreply.github.com> Date: Tue, 17 Jun 2025 14:25:12 +0200 Subject: [PATCH 2/2] generate CDS for Dokka workers --- cdsScenarios.conf | 14 ++ .../gradle/build.gradle.kts | 7 +- .../kotlin/DatetimeGradleIntegrationTest.kt | 1 + .../dokka/it/AbstractIntegrationTest.kt | 3 +- .../dokka-gradle-plugin/build.gradle.kts | 7 + .../src/main/kotlin/internal/CdsSource.kt | 173 +++++++++++------- .../main/kotlin/tasks/DokkaGenerateTask.kt | 22 ++- dokka-runners/runner-cli/build.gradle.kts | 6 + 8 files changed, 155 insertions(+), 78 deletions(-) create mode 100644 cdsScenarios.conf diff --git a/cdsScenarios.conf b/cdsScenarios.conf new file mode 100644 index 0000000000..f1b3189fea --- /dev/null +++ b/cdsScenarios.conf @@ -0,0 +1,14 @@ + +a_testFunctional_noCds { + title = "testFunctional - No CDS" + tasks = [":dokka-gradle-plugin:testFunctional"] + gradle-args = ["-PenableDokkaCds=false"] + warm-ups = 2 +} + +b_testFunctional_withCds { + title = "testFunctional - With CDS" + tasks = [":dokka-gradle-plugin:testFunctional"] + gradle-args = ["-PenableDokkaCds=true"] + warm-ups = 2 +} diff --git a/dokka-integration-tests/gradle/build.gradle.kts b/dokka-integration-tests/gradle/build.gradle.kts index 94f3b6b512..9cbc54b26c 100644 --- a/dokka-integration-tests/gradle/build.gradle.kts +++ b/dokka-integration-tests/gradle/build.gradle.kts @@ -199,9 +199,10 @@ fun registerTestProjectSuite( .inputFile("templateSettingsGradleKts", templateSettingsGradleKts) .withPathSensitivity(NAME_ONLY) - if (jvm != null) { - javaLauncher = javaToolchains.launcherFor { languageVersion = jvm } - } +// if (jvm != null) { +// javaLauncher = javaToolchains.launcherFor { languageVersion = jvm } +// } + javaLauncher = javaToolchains.launcherFor { languageVersion = JavaLanguageVersion.of(17) } } } configure() diff --git a/dokka-integration-tests/gradle/src/testExternalProjectKotlinxDatetime/kotlin/DatetimeGradleIntegrationTest.kt b/dokka-integration-tests/gradle/src/testExternalProjectKotlinxDatetime/kotlin/DatetimeGradleIntegrationTest.kt index a94ee5a479..a7a67f9262 100644 --- a/dokka-integration-tests/gradle/src/testExternalProjectKotlinxDatetime/kotlin/DatetimeGradleIntegrationTest.kt +++ b/dokka-integration-tests/gradle/src/testExternalProjectKotlinxDatetime/kotlin/DatetimeGradleIntegrationTest.kt @@ -50,6 +50,7 @@ class DatetimeGradleIntegrationTest : AbstractGradleIntegrationTest(), TestOutpu @ParameterizedTest(name = "{0}") @ArgumentsSource(DatetimeBuildVersionsArgumentsProvider::class) fun execute(buildVersions: BuildVersions) { + println("TESTING PROJECT DIR " + projectDir.absoluteFile.invariantSeparatorsPath) val result = createGradleRunner(buildVersions, ":kotlinx-datetime:dokkaGenerate").buildRelaxed() assertEquals(TaskOutcome.SUCCESS, assertNotNull(result.task(":kotlinx-datetime:dokkaGenerate")).outcome) diff --git a/dokka-integration-tests/utilities/src/main/kotlin/org/jetbrains/dokka/it/AbstractIntegrationTest.kt b/dokka-integration-tests/utilities/src/main/kotlin/org/jetbrains/dokka/it/AbstractIntegrationTest.kt index 7c03c61417..96e2c44ed6 100644 --- a/dokka-integration-tests/utilities/src/main/kotlin/org/jetbrains/dokka/it/AbstractIntegrationTest.kt +++ b/dokka-integration-tests/utilities/src/main/kotlin/org/jetbrains/dokka/it/AbstractIntegrationTest.kt @@ -5,6 +5,7 @@ package org.jetbrains.dokka.it import org.jsoup.Jsoup +import org.junit.jupiter.api.io.CleanupMode import org.junit.jupiter.api.io.TempDir import java.io.File import java.net.URL @@ -15,7 +16,7 @@ import kotlin.test.assertTrue abstract class AbstractIntegrationTest { - @field:TempDir + @field:TempDir(cleanup = CleanupMode.NEVER) lateinit var tempFolder: File /** Working directory of the test. Contains the project that should be tested. */ diff --git a/dokka-runners/dokka-gradle-plugin/build.gradle.kts b/dokka-runners/dokka-gradle-plugin/build.gradle.kts index 44c83f763f..26054419a1 100644 --- a/dokka-runners/dokka-gradle-plugin/build.gradle.kts +++ b/dokka-runners/dokka-gradle-plugin/build.gradle.kts @@ -172,6 +172,13 @@ testing.suites { systemProperty("kotest.framework.config.fqn", "org.jetbrains.dokka.gradle.utils.KotestProjectConfig") // FIXME remove autoscan when Kotest >= 6.0 systemProperty("kotest.framework.classpath.scanning.autoscan.disable", "true") + + // cds requires java >= 17 + javaLauncher = javaToolchains.launcherFor { languageVersion = JavaLanguageVersion.of(17) } + + outputs.upToDateWhen { + !providers.gradleProperty("enableDokkaCds").isPresent + } } } } diff --git a/dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt index c8b60db66d..f46b52aa12 100644 --- a/dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt +++ b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/internal/CdsSource.kt @@ -5,8 +5,10 @@ package org.jetbrains.dokka.gradle.internal import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.logging.Logging import org.gradle.api.provider.ValueSource import org.gradle.api.provider.ValueSourceParameters +import org.gradle.kotlin.dsl.provideDelegate import org.gradle.process.ExecOperations import org.jetbrains.kotlin.konan.file.use import java.io.File @@ -24,6 +26,7 @@ import java.util.concurrent.locks.ReentrantLock import java.util.jar.JarFile import javax.inject.Inject import kotlin.concurrent.withLock +import kotlin.random.Random internal abstract class CdsSource @@ -40,39 +43,33 @@ internal constructor( checksum(parameters.classpath) } - private val cacheDir: File by lazy { - val osName = System.getProperty("os.name").lowercase() - val homeDir = System.getProperty("user.home") - val appDataDir = System.getenv("APP_DATA") ?: homeDir - - val userCacheDir = when { - "win" in osName -> "$appDataDir/Caches/" - "mac" in osName -> "$homeDir/Library/Caches/" - "nix" in osName -> "$homeDir/.cache/" - else -> "$homeDir/.cache/" - } - - File(userCacheDir).resolve("dokka").apply { - mkdirs() - } - } - private val cdsFile: File by lazy { - cacheDir.resolve("$classpathChecksum.jsa") + cdsCacheDir.resolve("$classpathChecksum.jsa") } private val lockFile: File by lazy { - cacheDir.resolve("$classpathChecksum.lock") + cdsCacheDir.resolve("$classpathChecksum.lock") } - override fun obtain(): File { + override fun obtain(): File? { + if (currentJavaVersion < 17) { + logger.warn("CDS generation is only supported for Java 17 and above. Current version $currentJavaVersion.") + return null + } + if (cdsFile.exists()) { + cdsFile.setLastModified(System.currentTimeMillis()) + return cdsFile + } + lock.withLock { RandomAccessFile(lockFile, "rw").use { - it.channel.lockWithRetries().use { - if (!cdsFile.exists()) { + it.channel.lockLenient().use { + if (cdsFile.exists()) { + return cdsFile + } else { generateStaticCds() + logger.warn("Using CDS ${cdsFile.absoluteFile.invariantSeparatorsPath}") + return cdsFile } - println("Using CDS ${cdsFile.absoluteFile.invariantSeparatorsPath}") - return cdsFile } } } @@ -80,28 +77,64 @@ internal constructor( private fun generateStaticCds() { - val classListFile = Files.createTempFile("asd", "classlist").toFile() - parameters.classpath.files.flatMap { file -> - getClassNamesFromJarFile(file) - } - .toSet() + val classListFile = Files.createTempFile("CdsSource", "classlist").toFile() + classListFile.deleteOnExit() + + parameters.classpath.files + .flatMap { file -> getClassNamesFromJarFile(file) } + .distinct() + .sorted() .joinToString("\n") .let { classListFile.writeText(it) } + logger.warn("Generating CDS from class list: ${classListFile.absoluteFile.invariantSeparatorsPath}") - execOps.javaexec { - jvmArgs( + execOps.exec { + executable("java") + args( "-Xshare:dump", "-XX:SharedArchiveFile=${cdsFile.absoluteFile.invariantSeparatorsPath}", - "-XX:SharedClassListFile=${classListFile.absoluteFile.invariantSeparatorsPath}" + "-XX:SharedClassListFile=${classListFile.absoluteFile.invariantSeparatorsPath}", + "-cp", + parameters.classpath.asPath, +// "${parameters.classpath.asPath}${File.pathSeparator}/Users/dev/projects/jetbrains/dokka/dokka-runners/runner-cli/build/libs/runner-cli-2.0.20-SNAPSHOT.jar", +// parameters.classpath.asPath, +// "org.jetbrains.dokka.MainKt" ) - classpath(parameters.classpath) + //logger.warn("Generating CDS args: $args") } } companion object { private val lock: Lock = ReentrantLock() + + private val logger = Logging.getLogger(CdsSource::class.java) + + private val cdsCacheDir: File by lazy { + val cdsFromEnv = System.getenv("DOKKA_CDS_CACHE_DIR") + + if (cdsFromEnv != null) { + File(cdsFromEnv).apply { + mkdirs() + } + } else { + val osName = System.getProperty("os.name").lowercase() + val homeDir = System.getProperty("user.home") + val appDataDir = System.getenv("APP_DATA") ?: homeDir + + val userCacheDir = when { + "win" in osName -> "$appDataDir/Caches/" + "mac" in osName -> "$homeDir/Library/Caches/" + "nix" in osName -> "$homeDir/.cache/" + else -> "$homeDir/.cache/" + } + + File(userCacheDir).resolve("dokka-cds").apply { + mkdirs() + } + } + } } } @@ -112,24 +145,18 @@ private fun checksum( val md = MessageDigest.getInstance("md5") DigestOutputStream(nullOutputStream(), md).use { os -> os.write(files.asPath.encodeToByteArray()) - } - return BigInteger(1, md.digest()).toString(16) - .padStart(md.digestLength * 2, '0') -} -private fun checksum( - files: Collection -): String { - val md = MessageDigest.getInstance("md5") - DigestOutputStream(nullOutputStream(), md).use { os -> files.forEach { file -> - file.inputStream().use { it.copyTo(os) } + file.inputStream().use { + it.copyTo(os) + } } } return BigInteger(1, md.digest()).toString(16) .padStart(md.digestLength * 2, '0') } + private fun nullOutputStream(): OutputStream = object : OutputStream() { override fun write(b: Int) {} @@ -149,30 +176,40 @@ private fun getClassNamesFromJarFile(source: File): Set { } } -private fun FileChannel.lockWithRetries(): FileLock { - var retries = 0 - while (true) { - try { - return lock() - } - /* - Catching the OverlappingFileLockException which is caused by the same jvm (process) already having locked the file. - Since we do use a static re-entrant lock as a monitor to the cache, this can only happen - when this code is running in the same JVM but with in complete isolation - (e.g. Gradle classpath isolation, or composite builds). - - If we detect this case, we retry the locking after a short period, constantly logging that we're blocked - by some other thread using the cache. - - The risk of deadlocking here is low, since we can only get into this code path, *if* - the code is very isolated and somebody locked the file. - */ - catch (t: OverlappingFileLockException) { - Thread.sleep(25) - retries++ -// if (retries % 10 == 0) { -//// logInfo("Waiting to acquire lock: $file") -// } - } +private val currentJavaVersion: Int = + System.getProperty("java.version") + .removePrefix("1.") + .substringBefore(".") + .toInt() + + +/** + * Leniently obtain a [FileLock] for the channel. + * + * @throws [InterruptedException] if the current thread is interrupted before the lock can be acquired. + */ +private tailrec fun FileChannel.lockLenient(): FileLock { + if (Thread.interrupted()) { + throw InterruptedException("Interrupted while waiting for lock on FileChannel@${this@lockLenient.hashCode()}") + } + + val lock = try { + tryLock() + } catch (_: OverlappingFileLockException) { + // ignore exception - it means the lock is already held by this process. + null } + + if (lock != null) { + return lock + } + + try { + Thread.sleep(Random.nextLong(25, 125)) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + throw e + } + + return lockLenient() } diff --git a/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt index 8e6ad0fbcb..dbb96747a7 100644 --- a/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt +++ b/dokka-runners/dokka-gradle-plugin/src/main/kotlin/tasks/DokkaGenerateTask.kt @@ -129,6 +129,12 @@ constructor( Publication, } + init { + outputs.upToDateWhen { + !providers.gradleProperty("enableDokkaCds").isPresent + } + } + @InternalDokkaGradlePluginApi protected fun generateDocumentation( generationType: GeneratorMode, @@ -168,14 +174,18 @@ constructor( isolation.jvmArgs.orNull?.filter { it.isNotBlank() }?.let(this::setJvmArgs) isolation.systemProperties.orNull?.let(this::systemProperties) - val cds = providers.of(CdsSource::class) { - parameters { - classpath.from(runtimeClasspath) + if (providers.gradleProperty("enableDokkaCds").orNull.toBoolean()) { + val cds = providers.of(CdsSource::class) { + parameters { + classpath.from(runtimeClasspath) + } + } + cds.orNull?.let { + jvmArgs( + "-XX:SharedArchiveFile=${it.absoluteFile.invariantSeparatorsPath}" + ) } } - jvmArgs( - "-XX:SharedArchiveFile=${cds.get().absoluteFile.invariantSeparatorsPath}" - ) } } } diff --git a/dokka-runners/runner-cli/build.gradle.kts b/dokka-runners/runner-cli/build.gradle.kts index 03842b5310..5daaca2740 100644 --- a/dokka-runners/runner-cli/build.gradle.kts +++ b/dokka-runners/runner-cli/build.gradle.kts @@ -24,3 +24,9 @@ tasks.shadowJar { attributes("Main-Class" to "org.jetbrains.dokka.MainKt") } } + +tasks.jar { + manifest { + attributes("Main-Class" to "org.jetbrains.dokka.MainKt") + } +}