Symbol graph support for swiftbuild build system (#8923)

Refactor the build system protocol to make symbol graphs and build plans
optional build outputs. Rewrite the dump-symbol-graph and the plugin
delegate to use this new protocol and get support from the swiftbuild build
system.

Author: cmcgee1024

Sources/Build/BuildOperation.swift

@@ -396,9 +396,11 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     }
 
     /// Perform a build using the given build description and subset.
-    public func build(subset: BuildSubset) async throws -> BuildResult {
+    public func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult {
+        var result = BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped")))
+
         guard !self.config.shouldSkipBuilding(for: .target) else {
-            return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped")))
+            return result
         }
 
         let buildStartTime = DispatchTime.now()
@@ -422,7 +424,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         // any errors up-front. Returns true if we should proceed with the build
         // or false if not. It will already have thrown any appropriate error.
         guard try await self.compilePlugins(in: subset) else {
-            return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Plugin compilation failed")))
+            result.serializedDiagnosticPathsByTargetName = .failure(StringError("Plugin compilation failed"))
+            return result
         }
 
         let configuration = self.config.configuration(for: .target)
@@ -452,15 +455,18 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         )
         guard success else { throw Diagnostics.fatalError }
 
-        let serializedDiagnosticResult: Result<[String: [AbsolutePath]], Error>
+        if buildOutputs.contains(.buildPlan) {
+            result.buildPlan = try buildPlan
+        }
+
         var serializedDiagnosticPaths: [String: [AbsolutePath]] = [:]
         do {
             for module in try buildPlan.buildModules {
                 serializedDiagnosticPaths[module.module.name, default: []].append(contentsOf: module.diagnosticFiles)
             }
-            serializedDiagnosticResult = .success(serializedDiagnosticPaths)
+            result.serializedDiagnosticPathsByTargetName = .success(serializedDiagnosticPaths)
         } catch {
-            serializedDiagnosticResult = .failure(error)
+            result.serializedDiagnosticPathsByTargetName = .failure(error)
         }
 
         // Create backwards-compatibility symlink to old build path.
@@ -474,7 +480,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                     warning: "unable to delete \(oldBuildPath), skip creating symbolic link",
                     underlyingError: error
                 )
-                return BuildResult(serializedDiagnosticPathsByTargetName: serializedDiagnosticResult)
+
+                return result
             }
         }
 
@@ -491,7 +498,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
             )
         }
 
-        return BuildResult(serializedDiagnosticPathsByTargetName: serializedDiagnosticResult)
+        return result
     }
 
     /// Compiles any plugins specified or implied by the build subset, returning

Sources/Commands/PackageCommands/APIDiff.swift

@@ -123,14 +123,17 @@ struct APIDiff: AsyncSwiftCommand {
         let apiDigesterTool = SwiftAPIDigester(fileSystem: swiftCommandState.fileSystem, tool: apiDigesterPath)
 
         // Build the current package.
-        try await buildSystem.build()
+        let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.buildPlan])
+        guard let buildPlan = buildResult.buildPlan else {
+            throw ExitCode.failure
+        }
 
         // Dump JSON for the baseline package.
         let baselineDumper = try APIDigesterBaselineDumper(
             baselineRevision: baselineRevision,
             packageRoot: swiftCommandState.getPackageRoot(),
-            productsBuildParameters: try buildSystem.buildPlan.destinationBuildParameters,
-            toolsBuildParameters: try buildSystem.buildPlan.toolsBuildParameters,
+            productsBuildParameters: buildPlan.destinationBuildParameters,
+            toolsBuildParameters: buildPlan.toolsBuildParameters,
             apiDigesterTool: apiDigesterTool,
             observabilityScope: swiftCommandState.observabilityScope
         )
@@ -159,7 +162,7 @@ struct APIDiff: AsyncSwiftCommand {
                         if let comparisonResult = try apiDigesterTool.compareAPIToBaseline(
                             at: moduleBaselinePath,
                             for: module,
-                            buildPlan: try buildSystem.buildPlan,
+                            buildPlan: buildPlan,
                             except: breakageAllowlistPath
                         ) {
                             return comparisonResult

Sources/Commands/PackageCommands/DumpCommands.swift

@@ -52,54 +52,79 @@ struct DumpSymbolGraph: AsyncSwiftCommand {
         //
         // We turn build manifest caching off because we need the build plan.
         let buildSystem = try await swiftCommandState.createBuildSystem(
-            explicitBuildSystem: .native,
             // We are enabling all traits for dumping the symbol graph.
             traitConfiguration: .init(enableAllTraits: true),
             cacheBuildManifest: false
         )
-        try await buildSystem.build()
+        // TODO pass along the various flags as associated values to the symbol graph build output (e.g. includeSPISymbols)
+        let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.symbolGraph, .buildPlan])
 
-        // Configure the symbol graph extractor.
-        let symbolGraphExtractor = try SymbolGraphExtract(
-            fileSystem: swiftCommandState.fileSystem,
-            tool: swiftCommandState.getTargetToolchain().getSymbolGraphExtract(),
-            observabilityScope: swiftCommandState.observabilityScope,
-            skipSynthesizedMembers: skipSynthesizedMembers,
-            minimumAccessLevel: minimumAccessLevel,
-            skipInheritedDocs: skipInheritedDocs,
-            includeSPISymbols: includeSPISymbols,
-            emitExtensionBlockSymbols: extensionBlockSymbolBehavior == .emitExtensionBlockSymbols,
-            outputFormat: .json(pretty: prettyPrint)
-        )
+        let symbolGraphDirectory = try swiftCommandState.productsBuildParameters.dataPath.appending("symbolgraph")
 
-        // Run the tool once for every library and executable target in the root package.
-        let buildPlan = try buildSystem.buildPlan
-        let modulesGraph = try await buildSystem.getPackageGraph()
-        let symbolGraphDirectory = buildPlan.destinationBuildParameters.dataPath.appending("symbolgraph")
-        for description in buildPlan.buildModules {
-            guard description.module.type == .library,
-                  modulesGraph.rootPackages[description.package.id] != nil
-            else {
-                continue
-            }
+        let fs = swiftCommandState.fileSystem
+
+        try? fs.removeFileTree(symbolGraphDirectory)
+        try fs.createDirectory(symbolGraphDirectory, recursive: true)
+
+        if let symbolGraph = buildResult.symbolGraph {
+            // The build system produced symbol graphs for us, one for each target.
+            let buildPath = try swiftCommandState.productsBuildParameters.buildPath
 
-            print("-- Emitting symbol graph for", description.module.name)
-            let result = try symbolGraphExtractor.extractSymbolGraph(
-                for: description,
-                outputRedirection: .collect(redirectStderr: true),
-                outputDirectory: symbolGraphDirectory,
-                verboseOutput: swiftCommandState.logLevel <= .info
+            // Copy the symbol graphs from the target-specific locations to the single output directory
+            for rootPackage in try await buildSystem.getPackageGraph().rootPackages {
+                for module in rootPackage.modules {
+                    let sgDir = symbolGraph.outputLocationForTarget(module.name, try swiftCommandState.productsBuildParameters)
+
+                    if case let sgDir = buildPath.appending(components: sgDir), fs.exists(sgDir) {
+                        for sgFile in try fs.getDirectoryContents(sgDir) {
+                            try fs.copy(from: sgDir.appending(components: sgFile), to: symbolGraphDirectory.appending(sgFile))
+                        }
+                    }
+                }
+            }
+        } else if let buildPlan = buildResult.buildPlan {
+            // Otherwise, with a build plan we can run the symbol graph extractor tool on the built targets.
+            let symbolGraphExtractor = try SymbolGraphExtract(
+                fileSystem: swiftCommandState.fileSystem,
+                tool: swiftCommandState.getTargetToolchain().getSymbolGraphExtract(),
+                observabilityScope: swiftCommandState.observabilityScope,
+                skipSynthesizedMembers: skipSynthesizedMembers,
+                minimumAccessLevel: minimumAccessLevel,
+                skipInheritedDocs: skipInheritedDocs,
+                includeSPISymbols: includeSPISymbols,
+                emitExtensionBlockSymbols: extensionBlockSymbolBehavior == .emitExtensionBlockSymbols,
+                outputFormat: .json(pretty: prettyPrint)
             )
 
-            if result.exitStatus != .terminated(code: 0) {
-                let commandline = "\nUsing commandline: \(result.arguments)"
-                switch result.output {
-                case .success(let value):
-                    swiftCommandState.observabilityScope.emit(error: "Failed to emit symbol graph for '\(description.module.c99name)': \(String(decoding: value, as: UTF8.self))\(commandline)")
-                case .failure(let error):
-                    swiftCommandState.observabilityScope.emit(error: "Internal error while emitting symbol graph for '\(description.module.c99name)': \(error)\(commandline)")
+            // Run the tool once for every library and executable target in the root package.
+            let modulesGraph = try await buildSystem.getPackageGraph()
+            for description in buildPlan.buildModules {
+                guard description.module.type == .library,
+                    modulesGraph.rootPackages[description.package.id] != nil
+                else {
+                    continue
+                }
+
+                print("-- Emitting symbol graph for", description.module.name)
+                let result = try symbolGraphExtractor.extractSymbolGraph(
+                    for: description,
+                    outputRedirection: .collect(redirectStderr: true),
+                    outputDirectory: symbolGraphDirectory,
+                    verboseOutput: swiftCommandState.logLevel <= .info
+                )
+
+                if result.exitStatus != .terminated(code: 0) {
+                    let commandline = "\nUsing commandline: \(result.arguments)"
+                    switch result.output {
+                    case .success(let value):
+                        swiftCommandState.observabilityScope.emit(error: "Failed to emit symbol graph for '\(description.module.c99name)': \(String(decoding: value, as: UTF8.self))\(commandline)")
+                    case .failure(let error):
+                        swiftCommandState.observabilityScope.emit(error: "Internal error while emitting symbol graph for '\(description.module.c99name)': \(error)\(commandline)")
+                    }
                 }
             }
+        } else {
+            throw InternalError("Build system \(buildSystem) cannot produce a symbol graph.")
         }
 
         print("Files written to", symbolGraphDirectory.pathString)

Sources/Commands/PackageCommands/Install.swift

@@ -89,7 +89,7 @@ extension SwiftPackageCommand {
             }
 
             try await commandState.createBuildSystem(explicitProduct: productToInstall.name, traitConfiguration: .init())
-                .build(subset: .product(productToInstall.name))
+                .build(subset: .product(productToInstall.name), buildOutputs: [])
 
             let binPath = try commandState.productsBuildParameters.buildPath.appending(component: productToInstall.name)
             let finalBinPath = swiftpmBinDir.appending(component: binPath.basename)

Sources/Commands/PackageCommands/Migrate.swift

@@ -101,14 +101,15 @@ extension SwiftPackageCommand {
             // Next, let's build all of the individual targets or the
             // whole project to get diagnostic files.
             print("> Starting the build")
+
             var diagnosticsPaths: [String: [AbsolutePath]] = [:]
             if !targets.isEmpty {
                 for target in targets {
-                    let buildResult = try await buildSystem.build(subset: .target(target))
+                    let buildResult = try await buildSystem.build(subset: .target(target), buildOutputs: [])
                     diagnosticsPaths.merge(try buildResult.serializedDiagnosticPathsByTargetName.get(), uniquingKeysWith: { $0 + $1 })
                 }
             } else {
-                diagnosticsPaths = try await buildSystem.build(subset: .allIncludingTests).serializedDiagnosticPathsByTargetName.get()
+                diagnosticsPaths = try await buildSystem.build(subset: .allIncludingTests, buildOutputs: []).serializedDiagnosticPathsByTargetName.get()
             }
 
             var summary = SwiftFixIt.Summary(numberOfFixItsApplied: 0, numberOfFilesChanged: 0)

Sources/Commands/PackageCommands/PluginCommand.swift

@@ -351,11 +351,10 @@ struct PluginCommand: AsyncSwiftCommand {
             for: try pluginScriptRunner.hostTriple
         ) { name, path in
             // Build the product referenced by the tool, and add the executable to the tool map. Product dependencies are not supported within a package, so if the tool happens to be from the same package, we instead find the executable that corresponds to the product. There is always one, because of autogeneration of implicit executables with the same name as the target if there isn't an explicit one.
-            try await buildSystem.build(subset: .product(name, for: .host))
+            let buildResult = try await buildSystem.build(subset: .product(name, for: .host), buildOutputs: [.buildPlan])
 
-            // TODO determine if there is a common way to calculate the build tool binary path that doesn't depend on the build system.
-            if buildSystemKind == .native {
-                if let builtTool = try buildSystem.buildPlan.buildProducts.first(where: {
+            if let buildPlan = buildResult.buildPlan {
+                if let builtTool = buildPlan.buildProducts.first(where: {
                     $0.product.name == name && $0.buildParameters.destination == .host
                 }) {
                     return try builtTool.binaryPath

Sources/Commands/Snippets/Cards/SnippetCard.swift

@@ -113,7 +113,7 @@ struct SnippetCard: Card {
     func runExample() async throws {
         print("Building '\(snippet.path)'\n")
         let buildSystem = try await swiftCommandState.createBuildSystem(explicitProduct: snippet.name, traitConfiguration: .init())
-        try await buildSystem.build(subset: .product(snippet.name))
+        try await buildSystem.build(subset: .product(snippet.name), buildOutputs: [])
         let executablePath = try swiftCommandState.productsBuildParameters.buildPath.appending(component: snippet.name)
         if let exampleTarget = try await buildSystem.getPackageGraph().module(for: snippet.name) {
             try ProcessEnv.chdir(exampleTarget.sources.paths[0].parentDirectory)

Sources/Commands/SwiftBuildCommand.swift

@@ -203,7 +203,7 @@ public struct SwiftBuildCommand: AsyncSwiftCommand {
             outputStream: TSCBasic.stdoutStream
         )
         do {
-            try await buildSystem.build(subset: subset)
+            try await buildSystem.build(subset: subset, buildOutputs: [])
         } catch _ as Diagnostics {
             throw ExitCode.failure
         }

Sources/Commands/SwiftRunCommand.swift

@@ -145,10 +145,13 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
             )
 
             // Perform build.
-            try await buildSystem.build()
+            let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.buildPlan])
+            guard let buildPlan = buildResult.buildPlan else {
+                throw ExitCode.failure
+            }
 
             // Execute the REPL.
-            let arguments = try buildSystem.buildPlan.createREPLArguments()
+            let arguments = try buildPlan.createREPLArguments()
             print("Launching Swift REPL with arguments: \(arguments.joined(separator: " "))")
             try self.run(
                 fileSystem: swiftCommandState.fileSystem,
@@ -165,9 +168,9 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
                 )
                 let productName = try await findProductName(in: buildSystem.getPackageGraph())
                 if options.shouldBuildTests {
-                    try await buildSystem.build(subset: .allIncludingTests)
+                    try await buildSystem.build(subset: .allIncludingTests, buildOutputs: [])
                 } else if options.shouldBuild {
-                    try await buildSystem.build(subset: .product(productName))
+                    try await buildSystem.build(subset: .product(productName), buildOutputs: [])
                 }
 
                 let productRelativePath = try swiftCommandState.productsBuildParameters.executablePath(for: productName)
@@ -221,9 +224,9 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
                 )
                 let productName = try await findProductName(in: buildSystem.getPackageGraph())
                 if options.shouldBuildTests {
-                    try await buildSystem.build(subset: .allIncludingTests)
+                    try await buildSystem.build(subset: .allIncludingTests, buildOutputs: [])
                 } else if options.shouldBuild {
-                    try await buildSystem.build(subset: .product(productName))
+                    try await buildSystem.build(subset: .product(productName), buildOutputs: [])
                 }
 
                 let executablePath = try swiftCommandState.productsBuildParameters.buildPath.appending(component: productName)

Sources/Commands/SwiftTestCommand.swift

@@ -1572,7 +1572,7 @@ private func buildTestsIfNeeded(
         .allIncludingTests
     }
 
-    try await buildSystem.build(subset: subset)
+    try await buildSystem.build(subset: subset, buildOutputs: [])
 
     // Find the test product.
     let testProducts = await buildSystem.builtTestProducts