diff --git a/Sources/Build/BuildOperation.swift b/Sources/Build/BuildOperation.swift index 747c1ef7e44..2b341cc1c2d 100644 --- a/Sources/Build/BuildOperation.swift +++ b/Sources/Build/BuildOperation.swift @@ -396,9 +396,11 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS } /// Perform a build using the given build description and subset. - public func build(subset: BuildSubset) async throws -> BuildResult { + public func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult { + var result = BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped"))) + guard !self.config.shouldSkipBuilding(for: .target) else { - return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped"))) + return result } let buildStartTime = DispatchTime.now() @@ -422,7 +424,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS // any errors up-front. Returns true if we should proceed with the build // or false if not. It will already have thrown any appropriate error. guard try await self.compilePlugins(in: subset) else { - return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Plugin compilation failed"))) + result.serializedDiagnosticPathsByTargetName = .failure(StringError("Plugin compilation failed")) + return result } let configuration = self.config.configuration(for: .target) @@ -452,15 +455,18 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS ) guard success else { throw Diagnostics.fatalError } - let serializedDiagnosticResult: Result<[String: [AbsolutePath]], Error> + if buildOutputs.contains(.buildPlan) { + result.buildPlan = try buildPlan + } + var serializedDiagnosticPaths: [String: [AbsolutePath]] = [:] do { for module in try buildPlan.buildModules { serializedDiagnosticPaths[module.module.name, default: []].append(contentsOf: module.diagnosticFiles) } - serializedDiagnosticResult = .success(serializedDiagnosticPaths) + result.serializedDiagnosticPathsByTargetName = .success(serializedDiagnosticPaths) } catch { - serializedDiagnosticResult = .failure(error) + result.serializedDiagnosticPathsByTargetName = .failure(error) } // Create backwards-compatibility symlink to old build path. @@ -474,7 +480,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS warning: "unable to delete \(oldBuildPath), skip creating symbolic link", underlyingError: error ) - return BuildResult(serializedDiagnosticPathsByTargetName: serializedDiagnosticResult) + + return result } } @@ -491,7 +498,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS ) } - return BuildResult(serializedDiagnosticPathsByTargetName: serializedDiagnosticResult) + return result } /// Compiles any plugins specified or implied by the build subset, returning diff --git a/Sources/Commands/PackageCommands/APIDiff.swift b/Sources/Commands/PackageCommands/APIDiff.swift index 9b380d90e9d..856d90e7e0e 100644 --- a/Sources/Commands/PackageCommands/APIDiff.swift +++ b/Sources/Commands/PackageCommands/APIDiff.swift @@ -123,14 +123,17 @@ struct APIDiff: AsyncSwiftCommand { let apiDigesterTool = SwiftAPIDigester(fileSystem: swiftCommandState.fileSystem, tool: apiDigesterPath) // Build the current package. - try await buildSystem.build() + let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.buildPlan]) + guard let buildPlan = buildResult.buildPlan else { + throw ExitCode.failure + } // Dump JSON for the baseline package. let baselineDumper = try APIDigesterBaselineDumper( baselineRevision: baselineRevision, packageRoot: swiftCommandState.getPackageRoot(), - productsBuildParameters: try buildSystem.buildPlan.destinationBuildParameters, - toolsBuildParameters: try buildSystem.buildPlan.toolsBuildParameters, + productsBuildParameters: buildPlan.destinationBuildParameters, + toolsBuildParameters: buildPlan.toolsBuildParameters, apiDigesterTool: apiDigesterTool, observabilityScope: swiftCommandState.observabilityScope ) @@ -159,7 +162,7 @@ struct APIDiff: AsyncSwiftCommand { if let comparisonResult = try apiDigesterTool.compareAPIToBaseline( at: moduleBaselinePath, for: module, - buildPlan: try buildSystem.buildPlan, + buildPlan: buildPlan, except: breakageAllowlistPath ) { return comparisonResult diff --git a/Sources/Commands/PackageCommands/DumpCommands.swift b/Sources/Commands/PackageCommands/DumpCommands.swift index e5ccf3873fe..f3bb785f487 100644 --- a/Sources/Commands/PackageCommands/DumpCommands.swift +++ b/Sources/Commands/PackageCommands/DumpCommands.swift @@ -52,54 +52,79 @@ struct DumpSymbolGraph: AsyncSwiftCommand { // // We turn build manifest caching off because we need the build plan. let buildSystem = try await swiftCommandState.createBuildSystem( - explicitBuildSystem: .native, // We are enabling all traits for dumping the symbol graph. traitConfiguration: .init(enableAllTraits: true), cacheBuildManifest: false ) - try await buildSystem.build() + // TODO pass along the various flags as associated values to the symbol graph build output (e.g. includeSPISymbols) + let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.symbolGraph, .buildPlan]) - // Configure the symbol graph extractor. - let symbolGraphExtractor = try SymbolGraphExtract( - fileSystem: swiftCommandState.fileSystem, - tool: swiftCommandState.getTargetToolchain().getSymbolGraphExtract(), - observabilityScope: swiftCommandState.observabilityScope, - skipSynthesizedMembers: skipSynthesizedMembers, - minimumAccessLevel: minimumAccessLevel, - skipInheritedDocs: skipInheritedDocs, - includeSPISymbols: includeSPISymbols, - emitExtensionBlockSymbols: extensionBlockSymbolBehavior == .emitExtensionBlockSymbols, - outputFormat: .json(pretty: prettyPrint) - ) + let symbolGraphDirectory = try swiftCommandState.productsBuildParameters.dataPath.appending("symbolgraph") - // Run the tool once for every library and executable target in the root package. - let buildPlan = try buildSystem.buildPlan - let modulesGraph = try await buildSystem.getPackageGraph() - let symbolGraphDirectory = buildPlan.destinationBuildParameters.dataPath.appending("symbolgraph") - for description in buildPlan.buildModules { - guard description.module.type == .library, - modulesGraph.rootPackages[description.package.id] != nil - else { - continue - } + let fs = swiftCommandState.fileSystem + + try? fs.removeFileTree(symbolGraphDirectory) + try fs.createDirectory(symbolGraphDirectory, recursive: true) + + if let symbolGraph = buildResult.symbolGraph { + // The build system produced symbol graphs for us, one for each target. + let buildPath = try swiftCommandState.productsBuildParameters.buildPath - print("-- Emitting symbol graph for", description.module.name) - let result = try symbolGraphExtractor.extractSymbolGraph( - for: description, - outputRedirection: .collect(redirectStderr: true), - outputDirectory: symbolGraphDirectory, - verboseOutput: swiftCommandState.logLevel <= .info + // Copy the symbol graphs from the target-specific locations to the single output directory + for rootPackage in try await buildSystem.getPackageGraph().rootPackages { + for module in rootPackage.modules { + let sgDir = symbolGraph.outputLocationForTarget(module.name, try swiftCommandState.productsBuildParameters) + + if case let sgDir = buildPath.appending(components: sgDir), fs.exists(sgDir) { + for sgFile in try fs.getDirectoryContents(sgDir) { + try fs.copy(from: sgDir.appending(components: sgFile), to: symbolGraphDirectory.appending(sgFile)) + } + } + } + } + } else if let buildPlan = buildResult.buildPlan { + // Otherwise, with a build plan we can run the symbol graph extractor tool on the built targets. + let symbolGraphExtractor = try SymbolGraphExtract( + fileSystem: swiftCommandState.fileSystem, + tool: swiftCommandState.getTargetToolchain().getSymbolGraphExtract(), + observabilityScope: swiftCommandState.observabilityScope, + skipSynthesizedMembers: skipSynthesizedMembers, + minimumAccessLevel: minimumAccessLevel, + skipInheritedDocs: skipInheritedDocs, + includeSPISymbols: includeSPISymbols, + emitExtensionBlockSymbols: extensionBlockSymbolBehavior == .emitExtensionBlockSymbols, + outputFormat: .json(pretty: prettyPrint) ) - if result.exitStatus != .terminated(code: 0) { - let commandline = "\nUsing commandline: \(result.arguments)" - switch result.output { - case .success(let value): - swiftCommandState.observabilityScope.emit(error: "Failed to emit symbol graph for '\(description.module.c99name)': \(String(decoding: value, as: UTF8.self))\(commandline)") - case .failure(let error): - swiftCommandState.observabilityScope.emit(error: "Internal error while emitting symbol graph for '\(description.module.c99name)': \(error)\(commandline)") + // Run the tool once for every library and executable target in the root package. + let modulesGraph = try await buildSystem.getPackageGraph() + for description in buildPlan.buildModules { + guard description.module.type == .library, + modulesGraph.rootPackages[description.package.id] != nil + else { + continue + } + + print("-- Emitting symbol graph for", description.module.name) + let result = try symbolGraphExtractor.extractSymbolGraph( + for: description, + outputRedirection: .collect(redirectStderr: true), + outputDirectory: symbolGraphDirectory, + verboseOutput: swiftCommandState.logLevel <= .info + ) + + if result.exitStatus != .terminated(code: 0) { + let commandline = "\nUsing commandline: \(result.arguments)" + switch result.output { + case .success(let value): + swiftCommandState.observabilityScope.emit(error: "Failed to emit symbol graph for '\(description.module.c99name)': \(String(decoding: value, as: UTF8.self))\(commandline)") + case .failure(let error): + swiftCommandState.observabilityScope.emit(error: "Internal error while emitting symbol graph for '\(description.module.c99name)': \(error)\(commandline)") + } } } + } else { + throw InternalError("Build system \(buildSystem) cannot produce a symbol graph.") } print("Files written to", symbolGraphDirectory.pathString) diff --git a/Sources/Commands/PackageCommands/Install.swift b/Sources/Commands/PackageCommands/Install.swift index 9572c402559..900839fd971 100644 --- a/Sources/Commands/PackageCommands/Install.swift +++ b/Sources/Commands/PackageCommands/Install.swift @@ -89,7 +89,7 @@ extension SwiftPackageCommand { } try await commandState.createBuildSystem(explicitProduct: productToInstall.name, traitConfiguration: .init()) - .build(subset: .product(productToInstall.name)) + .build(subset: .product(productToInstall.name), buildOutputs: []) let binPath = try commandState.productsBuildParameters.buildPath.appending(component: productToInstall.name) let finalBinPath = swiftpmBinDir.appending(component: binPath.basename) diff --git a/Sources/Commands/PackageCommands/Migrate.swift b/Sources/Commands/PackageCommands/Migrate.swift index 0630e2d4642..094c0afb998 100644 --- a/Sources/Commands/PackageCommands/Migrate.swift +++ b/Sources/Commands/PackageCommands/Migrate.swift @@ -101,14 +101,15 @@ extension SwiftPackageCommand { // Next, let's build all of the individual targets or the // whole project to get diagnostic files. print("> Starting the build") + var diagnosticsPaths: [String: [AbsolutePath]] = [:] if !targets.isEmpty { for target in targets { - let buildResult = try await buildSystem.build(subset: .target(target)) + let buildResult = try await buildSystem.build(subset: .target(target), buildOutputs: []) diagnosticsPaths.merge(try buildResult.serializedDiagnosticPathsByTargetName.get(), uniquingKeysWith: { $0 + $1 }) } } else { - diagnosticsPaths = try await buildSystem.build(subset: .allIncludingTests).serializedDiagnosticPathsByTargetName.get() + diagnosticsPaths = try await buildSystem.build(subset: .allIncludingTests, buildOutputs: []).serializedDiagnosticPathsByTargetName.get() } var summary = SwiftFixIt.Summary(numberOfFixItsApplied: 0, numberOfFilesChanged: 0) diff --git a/Sources/Commands/PackageCommands/PluginCommand.swift b/Sources/Commands/PackageCommands/PluginCommand.swift index ec2243a6a71..40a923de710 100644 --- a/Sources/Commands/PackageCommands/PluginCommand.swift +++ b/Sources/Commands/PackageCommands/PluginCommand.swift @@ -351,11 +351,10 @@ struct PluginCommand: AsyncSwiftCommand { for: try pluginScriptRunner.hostTriple ) { name, path in // Build the product referenced by the tool, and add the executable to the tool map. Product dependencies are not supported within a package, so if the tool happens to be from the same package, we instead find the executable that corresponds to the product. There is always one, because of autogeneration of implicit executables with the same name as the target if there isn't an explicit one. - try await buildSystem.build(subset: .product(name, for: .host)) + let buildResult = try await buildSystem.build(subset: .product(name, for: .host), buildOutputs: [.buildPlan]) - // TODO determine if there is a common way to calculate the build tool binary path that doesn't depend on the build system. - if buildSystemKind == .native { - if let builtTool = try buildSystem.buildPlan.buildProducts.first(where: { + if let buildPlan = buildResult.buildPlan { + if let builtTool = buildPlan.buildProducts.first(where: { $0.product.name == name && $0.buildParameters.destination == .host }) { return try builtTool.binaryPath diff --git a/Sources/Commands/Snippets/Cards/SnippetCard.swift b/Sources/Commands/Snippets/Cards/SnippetCard.swift index d54b916ccf3..0810efaf92c 100644 --- a/Sources/Commands/Snippets/Cards/SnippetCard.swift +++ b/Sources/Commands/Snippets/Cards/SnippetCard.swift @@ -113,7 +113,7 @@ struct SnippetCard: Card { func runExample() async throws { print("Building '\(snippet.path)'\n") let buildSystem = try await swiftCommandState.createBuildSystem(explicitProduct: snippet.name, traitConfiguration: .init()) - try await buildSystem.build(subset: .product(snippet.name)) + try await buildSystem.build(subset: .product(snippet.name), buildOutputs: []) let executablePath = try swiftCommandState.productsBuildParameters.buildPath.appending(component: snippet.name) if let exampleTarget = try await buildSystem.getPackageGraph().module(for: snippet.name) { try ProcessEnv.chdir(exampleTarget.sources.paths[0].parentDirectory) diff --git a/Sources/Commands/SwiftBuildCommand.swift b/Sources/Commands/SwiftBuildCommand.swift index 8bc8e1aaccb..361085f85f6 100644 --- a/Sources/Commands/SwiftBuildCommand.swift +++ b/Sources/Commands/SwiftBuildCommand.swift @@ -203,7 +203,7 @@ public struct SwiftBuildCommand: AsyncSwiftCommand { outputStream: TSCBasic.stdoutStream ) do { - try await buildSystem.build(subset: subset) + try await buildSystem.build(subset: subset, buildOutputs: []) } catch _ as Diagnostics { throw ExitCode.failure } diff --git a/Sources/Commands/SwiftRunCommand.swift b/Sources/Commands/SwiftRunCommand.swift index ac08161af22..e15e95ee57e 100644 --- a/Sources/Commands/SwiftRunCommand.swift +++ b/Sources/Commands/SwiftRunCommand.swift @@ -145,10 +145,13 @@ public struct SwiftRunCommand: AsyncSwiftCommand { ) // Perform build. - try await buildSystem.build() + let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.buildPlan]) + guard let buildPlan = buildResult.buildPlan else { + throw ExitCode.failure + } // Execute the REPL. - let arguments = try buildSystem.buildPlan.createREPLArguments() + let arguments = try buildPlan.createREPLArguments() print("Launching Swift REPL with arguments: \(arguments.joined(separator: " "))") try self.run( fileSystem: swiftCommandState.fileSystem, @@ -165,9 +168,9 @@ public struct SwiftRunCommand: AsyncSwiftCommand { ) let productName = try await findProductName(in: buildSystem.getPackageGraph()) if options.shouldBuildTests { - try await buildSystem.build(subset: .allIncludingTests) + try await buildSystem.build(subset: .allIncludingTests, buildOutputs: []) } else if options.shouldBuild { - try await buildSystem.build(subset: .product(productName)) + try await buildSystem.build(subset: .product(productName), buildOutputs: []) } let productRelativePath = try swiftCommandState.productsBuildParameters.executablePath(for: productName) @@ -221,9 +224,9 @@ public struct SwiftRunCommand: AsyncSwiftCommand { ) let productName = try await findProductName(in: buildSystem.getPackageGraph()) if options.shouldBuildTests { - try await buildSystem.build(subset: .allIncludingTests) + try await buildSystem.build(subset: .allIncludingTests, buildOutputs: []) } else if options.shouldBuild { - try await buildSystem.build(subset: .product(productName)) + try await buildSystem.build(subset: .product(productName), buildOutputs: []) } let executablePath = try swiftCommandState.productsBuildParameters.buildPath.appending(component: productName) diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index a5b1e2cd919..a0bf4acaf0b 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -1572,7 +1572,7 @@ private func buildTestsIfNeeded( .allIncludingTests } - try await buildSystem.build(subset: subset) + try await buildSystem.build(subset: subset, buildOutputs: []) // Find the test product. let testProducts = await buildSystem.builtTestProducts diff --git a/Sources/Commands/Utilities/APIDigester.swift b/Sources/Commands/Utilities/APIDigester.swift index be402306492..ef5bb50bdce 100644 --- a/Sources/Commands/Utilities/APIDigester.swift +++ b/Sources/Commands/Utilities/APIDigester.swift @@ -146,7 +146,11 @@ struct APIDigesterBaselineDumper { toolsBuildParameters: toolsBuildParameters, packageGraphLoader: { graph } ) - try await buildSystem.build() + let buildResult = try await buildSystem.build(subset: .allExcludingTests, buildOutputs: [.buildPlan]) + + guard let buildPlan = buildResult.buildPlan else { + throw Diagnostics.fatalError + } // Dump the SDK JSON. try swiftCommandState.fileSystem.createDirectory(baselineDir, recursive: true) @@ -158,7 +162,7 @@ struct APIDigesterBaselineDumper { try apiDigesterTool.emitAPIBaseline( to: baselinePath(module), for: module, - buildPlan: buildSystem.buildPlan + buildPlan: buildPlan ) return nil } catch { diff --git a/Sources/Commands/Utilities/PluginDelegate.swift b/Sources/Commands/Utilities/PluginDelegate.swift index 98464b43227..c2843f0e5f0 100644 --- a/Sources/Commands/Utilities/PluginDelegate.swift +++ b/Sources/Commands/Utilities/PluginDelegate.swift @@ -249,7 +249,7 @@ final class PluginDelegate: PluginInvocationDelegate { traitConfiguration: .init(), toolsBuildParameters: toolsBuildParameters ) - try await buildSystem.build(subset: .allIncludingTests) + try await buildSystem.build(subset: .allIncludingTests, buildOutputs: []) // Clean out the code coverage directory that may contain stale `profraw` files from a previous run of // the code coverage tool. @@ -414,85 +414,93 @@ final class PluginDelegate: PluginInvocationDelegate { cacheBuildManifest: false ) - func lookupDescription( - for moduleName: String, - destination: BuildParameters.Destination - ) throws -> ModuleBuildDescription? { - try buildSystem.buildPlan.buildModules.first { - $0.module.name == moduleName && $0.buildParameters.destination == destination + // Build the target, if needed. We are interested in symbol graph (ideally) or a build plan. + // TODO pass along the options as associated values to the symbol graph build output (e.g. includeSPI) + let buildResult = try await buildSystem.build(subset: .target(targetName), buildOutputs: [.symbolGraph, .buildPlan]) + + if let symbolGraph = buildResult.symbolGraph { + let path = (try swiftCommandState.productsBuildParameters.buildPath) + return PluginInvocationSymbolGraphResult(directoryPath: "\(path)/\(symbolGraph.outputLocationForTarget(targetName, try swiftCommandState.productsBuildParameters).joined(separator:"/"))") + } else if let buildPlan = buildResult.buildPlan { + func lookupDescription( + for moduleName: String, + destination: BuildParameters.Destination + ) throws -> ModuleBuildDescription? { + try buildPlan.buildModules.first { + $0.module.name == moduleName && $0.buildParameters.destination == destination + } } - } - // Build the target, if needed. This would also create a build plan. - try await buildSystem.build(subset: .target(targetName)) - - // FIXME: The name alone doesn't give us enough information to figure out what - // the destination is, this logic prefers "target" over "host" because that's - // historically how this was setup. Ideally we should be building for both "host" - // and "target" if module is configured for them but that would require changing - // `PluginInvocationSymbolGraphResult` to carry multiple directories. - let description = if let targetDescription = try lookupDescription(for: targetName, destination: .target) { - targetDescription - } else if let hostDescription = try lookupDescription(for: targetName, destination: .host) { - hostDescription - } else { - throw InternalError("could not find a target named: \(targetName)") - } + // FIXME: The name alone doesn't give us enough information to figure out what + // the destination is, this logic prefers "target" over "host" because that's + // historically how this was setup. Ideally we should be building for both "host" + // and "target" if module is configured for them but that would require changing + // `PluginInvocationSymbolGraphResult` to carry multiple directories. + let description = if let targetDescription = try lookupDescription(for: targetName, destination: .target) { + targetDescription + } else if let hostDescription = try lookupDescription(for: targetName, destination: .host) { + hostDescription + } else { + throw InternalError("could not find a target named: \(targetName)") + } - // Configure the symbol graph extractor. - var symbolGraphExtractor = try SymbolGraphExtract( - fileSystem: swiftCommandState.fileSystem, - tool: swiftCommandState.getTargetToolchain().getSymbolGraphExtract(), - observabilityScope: swiftCommandState.observabilityScope - ) - symbolGraphExtractor.skipSynthesizedMembers = !options.includeSynthesized - switch options.minimumAccessLevel { - case .private: - symbolGraphExtractor.minimumAccessLevel = .private - case .fileprivate: - symbolGraphExtractor.minimumAccessLevel = .fileprivate - case .internal: - symbolGraphExtractor.minimumAccessLevel = .internal - case .package: - symbolGraphExtractor.minimumAccessLevel = .package - case .public: - symbolGraphExtractor.minimumAccessLevel = .public - case .open: - symbolGraphExtractor.minimumAccessLevel = .open - } - symbolGraphExtractor.skipInheritedDocs = true - symbolGraphExtractor.includeSPISymbols = options.includeSPI - symbolGraphExtractor.emitExtensionBlockSymbols = options.emitExtensionBlocks - - // Determine the output directory, and remove any old version if it already exists. - let outputDir = description.buildParameters.dataPath.appending( - components: "extracted-symbols", - description.package.identity.description, - targetName - ) - try swiftCommandState.fileSystem.removeFileTree(outputDir) - - // Run the symbol graph extractor on the target. - let result = try symbolGraphExtractor.extractSymbolGraph( - for: description, - outputRedirection: .collect, - outputDirectory: outputDir, - verboseOutput: self.swiftCommandState.logLevel <= .info - ) + // Configure the symbol graph extractor. + var symbolGraphExtractor = try SymbolGraphExtract( + fileSystem: swiftCommandState.fileSystem, + tool: swiftCommandState.getTargetToolchain().getSymbolGraphExtract(), + observabilityScope: swiftCommandState.observabilityScope + ) + symbolGraphExtractor.skipSynthesizedMembers = !options.includeSynthesized + switch options.minimumAccessLevel { + case .private: + symbolGraphExtractor.minimumAccessLevel = .private + case .fileprivate: + symbolGraphExtractor.minimumAccessLevel = .fileprivate + case .internal: + symbolGraphExtractor.minimumAccessLevel = .internal + case .package: + symbolGraphExtractor.minimumAccessLevel = .package + case .public: + symbolGraphExtractor.minimumAccessLevel = .public + case .open: + symbolGraphExtractor.minimumAccessLevel = .open + } + symbolGraphExtractor.skipInheritedDocs = true + symbolGraphExtractor.includeSPISymbols = options.includeSPI + symbolGraphExtractor.emitExtensionBlockSymbols = options.emitExtensionBlocks + + // Determine the output directory, and remove any old version if it already exists. + let outputDir = description.buildParameters.dataPath.appending( + components: "extracted-symbols", + description.package.identity.description, + targetName + ) + try swiftCommandState.fileSystem.removeFileTree(outputDir) + + // Run the symbol graph extractor on the target. + let result = try symbolGraphExtractor.extractSymbolGraph( + for: description, + outputRedirection: .collect, + outputDirectory: outputDir, + verboseOutput: self.swiftCommandState.logLevel <= .info + ) - guard result.exitStatus == .terminated(code: 0) else { - throw AsyncProcessResult.Error.nonZeroExit(result) - } + guard result.exitStatus == .terminated(code: 0) else { + throw AsyncProcessResult.Error.nonZeroExit(result) + } - // Return the results to the plugin. - return PluginInvocationSymbolGraphResult(directoryPath: outputDir.pathString) + // Return the results to the plugin. + return PluginInvocationSymbolGraphResult(directoryPath: outputDir.pathString) + } else { + throw InternalError("Build system \(buildSystem) doesn't have plugin support.") + } } } extension BuildSystem { fileprivate func buildIgnoringError(subset: BuildSubset) async -> Bool { do { - try await self.build(subset: subset) + try await self.build(subset: subset, buildOutputs: []) return true } catch { return false diff --git a/Sources/SPMBuildCore/BuildSystem/BuildSystem.swift b/Sources/SPMBuildCore/BuildSystem/BuildSystem.swift index 9eb77b58637..801d8795d03 100644 --- a/Sources/SPMBuildCore/BuildSystem/BuildSystem.swift +++ b/Sources/SPMBuildCore/BuildSystem/BuildSystem.swift @@ -34,6 +34,21 @@ public enum BuildSubset { case target(String, for: BuildParameters.Destination? = .none) } +/// Represents possible extra build outputs for a build. Some build systems +/// can produce certain extra outputs in the process of building. Not all +/// build systems can produce all possible build outputs. Check the build +/// result for indication that the output was produced. +public enum BuildOutput { + case symbolGraph + // TODO associated values for the following symbol graph options: + // "-skip-inherited-docs" + // "-symbol-graph-minimum-access-level", “” + // "-include-spi-symbols" + // "-emit-extension-block-symbols" + // "-emit-synthesized-members" + case buildPlan +} + /// A protocol that represents a build system used by SwiftPM for all build operations. This allows factoring out the /// implementation details between SwiftPM's `BuildOperation` and the Swift Build backed `SwiftBuildSystem`. public protocol BuildSystem: Cancellable { @@ -49,28 +64,42 @@ public protocol BuildSystem: Cancellable { /// Builds a subset of the package graph. /// - Parameters: - /// - subset: The subset of the package graph to build. + /// - buildOutputs: Additional build outputs requested from the build system. + /// - Returns: A build result with details about requested build and outputs. @discardableResult - func build(subset: BuildSubset) async throws -> BuildResult - - var buildPlan: BuildPlan { get throws } + func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult var hasIntegratedAPIDigesterSupport: Bool { get } } extension BuildSystem { - /// Builds the default subset: all targets excluding tests. + /// Builds the default subset: all targets excluding tests with no extra build outputs. @discardableResult public func build() async throws -> BuildResult { - try await build(subset: .allExcludingTests) + try await build(subset: .allExcludingTests, buildOutputs: []) } } +public struct SymbolGraphResult { + public init(outputLocationForTarget: @escaping (String, BuildParameters) -> [String]) { + self.outputLocationForTarget = outputLocationForTarget + } + + /// Find the build path relative location of the symbol graph output directory + /// for a provided target and build parameters. Note that the directory may not + /// exist when the target doesn't have any symbol graph output, as one example. + public let outputLocationForTarget: (String, BuildParameters) -> [String] +} + public struct BuildResult { - package init(serializedDiagnosticPathsByTargetName: Result<[String: [AbsolutePath]], Error>) { + package init(serializedDiagnosticPathsByTargetName: Result<[String: [AbsolutePath]], Error>, symbolGraph: SymbolGraphResult? = nil, buildPlan: BuildPlan? = nil) { self.serializedDiagnosticPathsByTargetName = serializedDiagnosticPathsByTargetName + self.symbolGraph = symbolGraph + self.buildPlan = buildPlan } + public var symbolGraph: SymbolGraphResult? + public var buildPlan: BuildPlan? public var serializedDiagnosticPathsByTargetName: Result<[String: [AbsolutePath]], Error> } diff --git a/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift b/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift index c8c4751ba8b..d750cb9695d 100644 --- a/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift +++ b/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift @@ -730,6 +730,8 @@ extension PackagePIFProjectBuilder { // Custom source module build settings, if any. pifBuilder.delegate.configureSourceModuleBuildSettings(sourceModule: sourceModule, settings: &settings) + settings[.SYMBOL_GRAPH_EXTRACTOR_OUTPUT_DIR] = "$(TARGET_BUILD_DIR)/$(CURRENT_ARCH)/\(sourceModule.name).symbolgraphs" + // Until this point the build settings for the target have been the same between debug and release // configurations. // The custom manifest settings might cause them to diverge. diff --git a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift index e759c5b4e12..4b88f73498b 100644 --- a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift +++ b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift @@ -281,17 +281,17 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { SwiftLanguageVersion.supportedSwiftLanguageVersions } - public func build(subset: BuildSubset) async throws -> BuildResult { + public func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult { guard !buildParameters.shouldSkipBuilding else { return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("Building was skipped"))) } try await writePIF(buildParameters: buildParameters) - return try await startSWBuildOperation(pifTargetName: subset.pifTargetName) + return try await startSWBuildOperation(pifTargetName: subset.pifTargetName, genSymbolGraph: buildOutputs.contains(.symbolGraph)) } - private func startSWBuildOperation(pifTargetName: String) async throws -> BuildResult { + private func startSWBuildOperation(pifTargetName: String, genSymbolGraph: Bool) async throws -> BuildResult { let buildStartTime = ContinuousClock.Instant.now return try await withService(connectionMode: .inProcessStatic(swiftbuildServiceEntryPoint)) { service in @@ -342,7 +342,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { throw error } - let request = try self.makeBuildRequest(configuredTargets: configuredTargets, derivedDataPath: derivedDataPath) + let request = try self.makeBuildRequest(configuredTargets: configuredTargets, derivedDataPath: derivedDataPath, genSymbolGraph: genSymbolGraph) struct BuildState { private var targetsByID: [Int: SwiftBuild.SwiftBuildMessage.TargetStartedInfo] = [:] @@ -511,7 +511,10 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { } catch { throw error } - return BuildResult(serializedDiagnosticPathsByTargetName: .success(serializedDiagnosticPathsByTargetName)) + + return BuildResult(serializedDiagnosticPathsByTargetName: .success(serializedDiagnosticPathsByTargetName), symbolGraph: SymbolGraphResult(outputLocationForTarget: { target, buildParameters in + return ["\(buildParameters.triple.archName)", "\(target).symbolgraphs"] + })) } } @@ -548,7 +551,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { ) } - private func makeBuildParameters() throws -> SwiftBuild.SWBBuildParameters { + private func makeBuildParameters(genSymbolGraph: Bool) throws -> SwiftBuild.SWBBuildParameters { // Generate the run destination parameters. let runDestination = makeRunDestination() @@ -566,6 +569,10 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { settings["SWIFT_EXEC"] = buildParameters.toolchain.swiftCompilerPath.pathString // FIXME: workaround for old Xcode installations such as what is in CI settings["LM_SKIP_METADATA_EXTRACTION"] = "YES" + if genSymbolGraph { + settings["RUN_SYMBOL_GRAPH_EXTRACT"] = "YES" + // TODO set additional symbol graph options from the build output here, such as "include-spi-symbols" + } let normalizedTriple = Triple(buildParameters.triple.triple, normalizing: true) if let deploymentTargetSettingName = normalizedTriple.deploymentTargetSettingName { @@ -630,9 +637,9 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { return params } - public func makeBuildRequest(configuredTargets: [SWBTargetGUID], derivedDataPath: Basics.AbsolutePath) throws -> SWBBuildRequest { + public func makeBuildRequest(configuredTargets: [SWBTargetGUID], derivedDataPath: Basics.AbsolutePath, genSymbolGraph: Bool) throws -> SWBBuildRequest { var request = SWBBuildRequest() - request.parameters = try makeBuildParameters() + request.parameters = try makeBuildParameters(genSymbolGraph: genSymbolGraph) request.configuredTargets = configuredTargets.map { SWBConfiguredTarget(guid: $0.rawValue, parameters: request.parameters) } request.useParallelTargets = true request.useImplicitDependencies = false diff --git a/Sources/XCBuildSupport/XcodeBuildSystem.swift b/Sources/XCBuildSupport/XcodeBuildSystem.swift index 0cf4eb014f6..74d89e4271d 100644 --- a/Sources/XCBuildSupport/XcodeBuildSystem.swift +++ b/Sources/XCBuildSupport/XcodeBuildSystem.swift @@ -160,9 +160,11 @@ public final class XcodeBuildSystem: SPMBuildCore.BuildSystem { return [] } - public func build(subset: BuildSubset) async throws -> BuildResult { + public func build(subset: BuildSubset, buildOutputs: [BuildOutput]) async throws -> BuildResult { + let buildResult = BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("XCBuild does not support reporting serialized diagnostics."))) + guard !buildParameters.shouldSkipBuilding else { - return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("XCBuild does not support reporting serialized diagnostics."))) + return buildResult } let pifBuilder = try await getPIFBuilder() @@ -244,12 +246,11 @@ public final class XcodeBuildSystem: SPMBuildCore.BuildSystem { throw Diagnostics.fatalError } - if !logLevel.isQuiet { - self.outputStream.send("Build complete!\n") - self.outputStream.flush() - } + guard !self.logLevel.isQuiet else { return buildResult } + self.outputStream.send("Build complete!\n") + self.outputStream.flush() - return BuildResult(serializedDiagnosticPathsByTargetName: .failure(StringError("XCBuild does not support reporting serialized diagnostics."))) + return buildResult } func createBuildParametersFile() throws -> AbsolutePath { diff --git a/Sources/swift-bootstrap/main.swift b/Sources/swift-bootstrap/main.swift index 0911ddee21c..7935ca9490b 100644 --- a/Sources/swift-bootstrap/main.swift +++ b/Sources/swift-bootstrap/main.swift @@ -266,7 +266,7 @@ struct SwiftBootstrapBuildTool: AsyncParsableCommand { shouldDisableLocalRpath: shouldDisableLocalRpath, logLevel: logLevel ) - try await buildSystem.build(subset: .allExcludingTests) + try await buildSystem.build(subset: .allExcludingTests, buildOutputs: []) } func createBuildSystem( diff --git a/Tests/CommandsTests/PackageCommandTests.swift b/Tests/CommandsTests/PackageCommandTests.swift index 97d84d10ff0..06e8e4f6e8d 100644 --- a/Tests/CommandsTests/PackageCommandTests.swift +++ b/Tests/CommandsTests/PackageCommandTests.swift @@ -559,7 +559,8 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase { func testDumpSymbolGraphCompactFormatting() async throws { // Depending on how the test is running, the `swift-symbolgraph-extract` tool might be unavailable. - try XCTSkipIf((try? UserToolchain.default.getSymbolGraphExtract()) == nil, "skipping test because the `swift-symbolgraph-extract` tools isn't available") + try XCTSkipIf(buildSystemProvider == .native && (try? UserToolchain.default.getSymbolGraphExtract()) == nil, "skipping test because the `swift-symbolgraph-extract` tools isn't available") + try XCTSkipIf(buildSystemProvider == .swiftbuild && ProcessInfo.hostOperatingSystem == .windows, "skipping test for Windows because of long file path issues") try await fixture(name: "DependencyResolution/Internal/Simple") { fixturePath in let compactGraphData = try await XCTAsyncUnwrap(await symbolGraph(atPath: fixturePath, withPrettyPrinting: false)) @@ -571,6 +572,7 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase { func testDumpSymbolGraphPrettyFormatting() async throws { // Depending on how the test is running, the `swift-symbolgraph-extract` tool might be unavailable. try XCTSkipIf((try? UserToolchain.default.getSymbolGraphExtract()) == nil, "skipping test because the `swift-symbolgraph-extract` tools isn't available") + try XCTSkipIf(buildSystemProvider == .swiftbuild, "skipping test because pretty printing isn't yet supported with swiftbuild build system via swift build and the swift compiler") try await fixture(name: "DependencyResolution/Internal/Simple") { fixturePath in let prettyGraphData = try await XCTAsyncUnwrap(await symbolGraph(atPath: fixturePath, withPrettyPrinting: true)) @@ -3390,16 +3392,25 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase { // Check that if we don't pass any target, we successfully get symbol graph information for all targets in the package, and at different paths. do { let (stdout, _) = try await self.execute(["generate-documentation"], packagePath: packageDir) - XCTAssertMatch(stdout, .and(.contains("MyLibrary:"), .contains(AbsolutePath("/mypackage/MyLibrary").pathString))) - XCTAssertMatch(stdout, .and(.contains("MyCommand:"), .contains(AbsolutePath("/mypackage/MyCommand").pathString))) - + if buildSystemProvider == .native { + XCTAssertMatch(stdout, .and(.contains("MyLibrary:"), .contains(AbsolutePath("/mypackage/MyLibrary").pathString))) + XCTAssertMatch(stdout, .and(.contains("MyCommand:"), .contains(AbsolutePath("/mypackage/MyCommand").pathString))) + } else if buildSystemProvider == .swiftbuild { + XCTAssertMatch(stdout, .and(.contains("MyLibrary:"), .contains(AbsolutePath("/MyLibrary.symbolgraphs").pathString))) + XCTAssertMatch(stdout, .and(.contains("MyCommand:"), .contains(AbsolutePath("/MyCommand.symbolgraphs").pathString))) + } } // Check that if we pass a target, we successfully get symbol graph information for just the target we asked for. do { let (stdout, _) = try await self.execute(["generate-documentation", "--target", "MyLibrary"], packagePath: packageDir) - XCTAssertMatch(stdout, .and(.contains("MyLibrary:"), .contains(AbsolutePath("/mypackage/MyLibrary").pathString))) - XCTAssertNoMatch(stdout, .and(.contains("MyCommand:"), .contains(AbsolutePath("/mypackage/MyCommand").pathString))) + if buildSystemProvider == .native { + XCTAssertMatch(stdout, .and(.contains("MyLibrary:"), .contains(AbsolutePath("/mypackage/MyLibrary").pathString))) + XCTAssertNoMatch(stdout, .and(.contains("MyCommand:"), .contains(AbsolutePath("/mypackage/MyCommand").pathString))) + } else if buildSystemProvider == .swiftbuild { + XCTAssertMatch(stdout, .and(.contains("MyLibrary:"), .contains(AbsolutePath("/MyLibrary.symbolgraphs").pathString))) + XCTAssertNoMatch(stdout, .and(.contains("MyCommand:"), .contains(AbsolutePath("/MyCommand.symbolgraphs").pathString))) + } } } } @@ -4166,7 +4177,8 @@ class PackageCommandSwiftBuildTests: PackageCommandTestCase { } override func testCommandPluginSymbolGraphCallbacks() async throws { - throw XCTSkip("SWBINTTODO: Symbol graph extraction does not yet work with swiftbuild build system") + try XCTSkipOnWindows(because: "TSCBasic/Path.swift:969: Assertion failed, https://github.com/swiftlang/swift-package-manager/issues/8602") + try await super.testCommandPluginSymbolGraphCallbacks() } override func testCommandPluginBuildingCallbacks() async throws { diff --git a/Tests/FunctionalTests/TraitTests.swift b/Tests/FunctionalTests/TraitTests.swift index eb1fe7e5940..e8eda347f88 100644 --- a/Tests/FunctionalTests/TraitTests.swift +++ b/Tests/FunctionalTests/TraitTests.swift @@ -560,24 +560,28 @@ struct TraitTests { buildSystem: BuildSystemProvider.Kind, configuration: BuildConfiguration, ) async throws { - try await fixture(name: "Traits") { fixturePath in - let (stdout, _) = try await executeSwiftPackage( - fixturePath.appending("Package10"), - configuration: configuration, - extraArgs: ["dump-symbol-graph", "--experimental-prune-unused-dependencies"], - buildSystem: buildSystem, - ) - let optionalPath = stdout - .lazy - .split(whereSeparator: \.isNewline) - .first { String($0).hasPrefix("Files written to ") }? - .dropFirst(17) + try await withKnownIssue(isIntermittent: true, { + try await fixture(name: "Traits") { fixturePath in + let (stdout, _) = try await executeSwiftPackage( + fixturePath.appending("Package10"), + configuration: configuration, + extraArgs: ["dump-symbol-graph", "--experimental-prune-unused-dependencies"], + buildSystem: buildSystem, + ) + let optionalPath = stdout + .lazy + .split(whereSeparator: \.isNewline) + .first { String($0).hasPrefix("Files written to ") }? + .dropFirst(17) - let path = try String(#require(optionalPath)) - let symbolGraph = try String(contentsOfFile: "\(path)/Package10Library1.symbols.json", encoding: .utf8) - #expect(symbolGraph.contains("TypeGatedByPackage10Trait1")) - #expect(symbolGraph.contains("TypeGatedByPackage10Trait2")) - } + let path = try String(#require(optionalPath)) + let symbolGraph = try String(contentsOfFile: "\(path)/Package10Library1.symbols.json", encoding: .utf8) + #expect(symbolGraph.contains("TypeGatedByPackage10Trait1")) + #expect(symbolGraph.contains("TypeGatedByPackage10Trait2")) + } + }, when: { + ProcessInfo.hostOperatingSystem == .windows + }) } @Test( @@ -595,7 +599,7 @@ struct TraitTests { // The swiftbuild build system doesn't yet have the ability for command plugins to request symbol graphs try await withKnownIssue( "https://github.com/swiftlang/swift-build/issues/609", - isIntermittent: (ProcessInfo.hostOperatingSystem == .windows), + isIntermittent: true, ) { let (stdout, _) = try await executeSwiftPackage( fixturePath.appending("Package10"), @@ -608,7 +612,7 @@ struct TraitTests { #expect(symbolGraph.contains("TypeGatedByPackage10Trait1")) #expect(symbolGraph.contains("TypeGatedByPackage10Trait2")) } when: { - buildSystem == .swiftbuild + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .windows } } }