From 58827c8d7683d32da0dc083186fd03be3edf67e5 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 22 Jul 2025 12:23:26 -0500 Subject: [PATCH 01/32] Demonstrate basic test discvoery using language server --- release/package.json | 5 ++ src/Components/TestExplorer.fs | 109 ++++++++++++++++++++++++++++++++- src/Core/DTO.fs | 17 +++++ src/Core/LanguageService.fs | 9 +++ 4 files changed, 137 insertions(+), 3 deletions(-) diff --git a/release/package.json b/release/package.json index 362d0a0d..5495efc2 100644 --- a/release/package.json +++ b/release/package.json @@ -901,6 +901,11 @@ "description": "Decides if the test explorer will automatically try discover tests when the workspace loads. You can still manually refresh the explorer to discover tests at any time", "type": "boolean" }, + "FSharp.TestExplorer.UseLegacyDotnetCliIntegration": { + "default": false, + "description": "Use the dotnet cli to discover and run tests instead of the language server. Will lose features like streamed test results and Microsoft Testing Platform support.", + "type": "boolean" + }, "FSharp.trace.server": { "default": "off", "description": "Trace server messages at the LSP protocol level for diagnostics.", diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 496623f7..54543f87 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -706,6 +706,17 @@ module TestItem = let getId (t: TestItem) = t.id + let tryPick (f: TestItem -> Option<'u>) root = + let rec recurse testItem = + let searchResult = f testItem + + if Option.isSome searchResult then + searchResult + else + testItem.children.TestItems() |> Array.tryPick recurse + + recurse root + let runnableChildren (root: TestItem) : TestItem array = // The goal is to collect here the actual runnable tests, they might be nested under a tree structure. let rec visit (testItem: TestItem) : TestItem array = @@ -1785,8 +1796,78 @@ module Mailbox = idleLoop () +module TestItemDTO = + + let testItemsOfTestDTOs testItemFactory tryGetLocation (flatTests: TestItemDTO array) = + + let fromTestItemDTO + (constructId: FullTestName -> TestId) + (itemFactory: TestItem.TestItemFactory) + (tryGetLocation: TestId -> LocationRecord option) + (hierarchy: TestName.NameHierarchy) + : TestItem = + let rec recurse (namedNode: TestName.NameHierarchy) = + let id = constructId namedNode.FullName + + let toUri path = + try + if String.IsNullOrEmpty path then + None + else + vscode.Uri.parse ($"file:///{path}", true) |> Some + with e -> + logger.Debug($"Failed to parse test location uri {path}", e) + None + + let toRange (rangeDto: TestFileRange) = + vscode.Range.Create( + vscode.Position.Create(rangeDto.StartLine, 0), + vscode.Position.Create(rangeDto.EndLine, 0) + ) + // TODO: figure out how to incorporate cached location data + itemFactory + { id = id + label = namedNode.Name + uri = namedNode.Data |> Option.bind (fun t -> t.CodeFilePath) |> Option.bind toUri + range = + namedNode.Data + |> Option.bind (fun t -> t.CodeLocationRange) + |> Option.map toRange + // uri = location |> LocationRecord.tryGetUri + // range = location |> LocationRecord.tryGetRange + children = namedNode.Children |> Array.map recurse + testFramework = + namedNode.Data + |> Option.bind (fun t -> t.ExecutorUri |> TrxParser.adapterTypeNameToTestFramework) } + + recurse hierarchy + + + let mapDtosForProject ((projectPath, targetFramework), flatTests) = + let namedHierarchies = + flatTests + |> Array.map (fun t -> {| Data = t; FullName = t.FullName |}) + |> TestName.inferHierarchy + + let projectChildTestItems = + namedHierarchies + |> Array.map (fromTestItemDTO (TestItem.constructId projectPath) testItemFactory tryGetLocation) + + TestItem.fromProject testItemFactory projectPath targetFramework projectChildTestItems + + let testDtosByProject = + flatTests |> Array.groupBy (fun dto -> dto.ProjectFilePath, dto.TargetFramework) + + let testItemsByProject = testDtosByProject |> Array.map mapDtosForProject + + testItemsByProject + + let activate (context: ExtensionContext) = + let useLegacyDotnetCliIntegration = + Configuration.get false "FSharp.TestExplorer.UseLegacyDotnetCliIntegration" + let testController = tests.createTestController ("fsharp-test-controller", "F# Test Controller") @@ -1839,9 +1920,31 @@ let activate (context: ExtensionContext) = let refreshHandler cancellationToken = - Interactions.refreshTestList testItemFactory testController.items tryGetLocation makeTrxPath cancellationToken - |> Promise.toThenable - |> (!^) + if useLegacyDotnetCliIntegration then + Interactions.refreshTestList + testItemFactory + testController.items + tryGetLocation + makeTrxPath + cancellationToken + |> Promise.toThenable + |> (!^) + else + promise { + try + let! discoveryResponse = LanguageService.testDiscovery () + + let testItems = + discoveryResponse.Data + |> TestItemDTO.testItemsOfTestDTOs testItemFactory tryGetLocation + |> ResizeArray + + testController.items.replace (testItems) + with e -> + logger.Error("Ionide test discovery threw an exception", e) + } + |> Promise.toThenable + |> (!^) testController.refreshHandler <- Some refreshHandler diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index 4be1a17e..c9216f28 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -368,6 +368,22 @@ module DTO = { file: string tests: TestAdapterEntry[] } + type TestFileRange = { StartLine: int; EndLine: int } + + type TestItemDTO = + { + FullName: string + DisplayName: string + /// Identifies the test adapter that ran the tests + /// Example: executor://xunit/VsTestRunner2/netcoreapp + /// Used for determining the test library, which effects how tests names are broken down + ExecutorUri: string + ProjectFilePath: string + TargetFramework: string + CodeFilePath: string option + CodeLocationRange: TestFileRange option + } + type Result<'T> = { Kind: string; Data: 'T } type HelptextResult = Result @@ -396,6 +412,7 @@ module DTO = type FSharpLiterateResult = Result type PipelineHintsResult = Result type TestResult = Result + type DiscoverTestsResult = Result module DotnetNew = diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index bcfa75fc..2b250fef 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -595,6 +595,15 @@ Consider: let fsiSdk () = promise { return Environment.configFsiSdkFilePath () } + let testDiscovery s = + match client with + | None -> + Promise.empty + | Some cl -> + cl.sendRequest ("test/discoverTests", ()) + |> Promise.map (fun (res: Types.PlainNotification) -> + res.content |> ofJson) + let private createClient (opts: Executable) = let options: ServerOptions = U5.Case2 {| run = opts; debug = opts |} From 128d6fa6f919ec81faab0c7afdf89d3e1fe8df09 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Fri, 25 Jul 2025 15:45:54 -0500 Subject: [PATCH 02/32] Use new test discovery in all places the old cli-based discovery was used Also refactor toward simpler swappability of the two approaches --- src/Components/TestExplorer.fs | 438 ++++++++++++++++++--------------- src/Core/LanguageService.fs | 6 +- 2 files changed, 235 insertions(+), 209 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 54543f87..ad23b061 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -848,6 +848,72 @@ module TestItem = children = children testFramework = None } + + let ofTestDTOs testItemFactory tryGetLocation (flatTests: TestItemDTO array) = + + let fromTestItemDTO + (constructId: FullTestName -> TestId) + (itemFactory: TestItemFactory) + (tryGetLocation: TestId -> LocationRecord option) + (hierarchy: TestName.NameHierarchy) + : TestItem = + let rec recurse (namedNode: TestName.NameHierarchy) = + let id = constructId namedNode.FullName + + let toUri path = + try + if String.IsNullOrEmpty path then + None + else + vscode.Uri.parse ($"file:///{path}", true) |> Some + with e -> + logger.Debug($"Failed to parse test location uri {path}", e) + None + + let toRange (rangeDto: TestFileRange) = + vscode.Range.Create( + vscode.Position.Create(rangeDto.StartLine, 0), + vscode.Position.Create(rangeDto.EndLine, 0) + ) + // TODO: figure out how to incorporate cached location data + itemFactory + { id = id + label = namedNode.Name + uri = namedNode.Data |> Option.bind (fun t -> t.CodeFilePath) |> Option.bind toUri + range = + namedNode.Data + |> Option.bind (fun t -> t.CodeLocationRange) + |> Option.map toRange + // uri = location |> LocationRecord.tryGetUri + // range = location |> LocationRecord.tryGetRange + children = namedNode.Children |> Array.map recurse + testFramework = + namedNode.Data + |> Option.bind (fun t -> t.ExecutorUri |> TrxParser.adapterTypeNameToTestFramework) } + + recurse hierarchy + + + let mapDtosForProject ((projectPath, targetFramework), flatTests) = + let namedHierarchies = + flatTests + |> Array.map (fun t -> {| Data = t; FullName = t.FullName |}) + |> TestName.inferHierarchy + + let projectChildTestItems = + namedHierarchies + |> Array.map (fromTestItemDTO (constructId projectPath) testItemFactory tryGetLocation) + + fromProject testItemFactory projectPath targetFramework projectChildTestItems + + let testDtosByProject = + flatTests |> Array.groupBy (fun dto -> dto.ProjectFilePath, dto.TargetFramework) + + let testItemsByProject = testDtosByProject |> Array.map mapDtosForProject + + testItemsByProject + + let isProjectItem (testId: TestId) = constructProjectRootId (getProjectPath testId) = testId @@ -1127,6 +1193,52 @@ module TestDiscovery = treeItems + let private tryInferTestFrameworkFromPackage (project: Project) = + + let detectablePackageToFramework = + dict + [ "Expecto", TestFrameworkId.Expecto + "xunit.abstractions", TestFrameworkId.XUnit ] + + let getPackageName (pr: PackageReference) = pr.Name + + project.PackageReferences + |> Array.tryPick (getPackageName >> Dict.tryGet detectablePackageToFramework) + + /// Does this project use a test framework where we can consistently discover test cases using `dotnet test --list-tests` + /// This requires the test library to print the fully-qualified test names + let canListTestCasesWithCli (project: Project) = + let librariesCapableOfListOnlyDiscovery = + set [ TestFrameworkId.Expecto; TestFrameworkId.XUnit ] + + tryInferTestFrameworkFromPackage project + |> Option.map librariesCapableOfListOnlyDiscovery.Contains + |> Option.defaultValue false + + + /// Use `dotnet test --list-tests` to + let discoverTestsByCliListTests testItemFactory tryGetLocation cancellationToken (project: Project) = + promise { + + let! testNames = DotnetCli.listTests project.Project project.Info.TargetFramework false cancellationToken + + let detectedTestFramework = tryInferTestFrameworkFromPackage project + + let testItemFactory (testItemBuilder: TestItem.TestItemBuilder) = + testItemFactory + { testItemBuilder with + testFramework = detectedTestFramework } + + let testHierarchy = + testNames + |> Array.map (fun n -> {| FullName = n; Data = () |}) + |> TestName.inferHierarchy + |> Array.map (TestItem.fromNamedHierarchy testItemFactory tryGetLocation project.Project) + + return TestItem.fromProject testItemFactory project.Project project.Info.TargetFramework testHierarchy + } + + module Interactions = type ProjectRunRequest = { @@ -1544,11 +1656,94 @@ module Interactions = } |> (Promise.toThenable >> (!^)) + let private discoverTestsWithDotnetCli + testItemFactory + tryGetLocation + makeTrxPath + report + (rootTestCollection: TestItemCollection) + cancellationToken + builtTestProjects + = + promise { + let warn (message: string) = + logger.Warn(message) + window.showWarningMessage (message) |> ignore + + let listDiscoveryProjects, trxDiscoveryProjects = + builtTestProjects |> List.partition TestDiscovery.canListTestCasesWithCli + + let discoverTestsByListOnly project = + report $"Discovering tests for {project.Project}" + TestDiscovery.discoverTestsByCliListTests testItemFactory tryGetLocation cancellationToken project + + let! listDiscoveredPerProject = + listDiscoveryProjects + |> ListExt.mapKeepInputAsync discoverTestsByListOnly + |> Promise.all + + trxDiscoveryProjects + |> List.iter (ProjectPath.fromProject >> makeTrxPath >> Path.deleteIfExists) + + let! _ = + trxDiscoveryProjects + |> Promise.executeWithMaxParallel maxParallelTestProjects (fun project -> + let projectPath = project.Project + report $"Discovering tests for {projectPath}" + let trxPath = makeTrxPath projectPath |> Some + + DotnetCli.test + projectPath + project.Info.TargetFramework + trxPath + None + DotnetCli.DebugTests.NoDebug + cancellationToken) + + let trxDiscoveredTests = + TestDiscovery.discoverFromTrx testItemFactory tryGetLocation makeTrxPath trxDiscoveryProjects + + + let listDiscoveredTests = listDiscoveredPerProject |> Array.map snd + let newTests = Array.concat [ listDiscoveredTests; trxDiscoveredTests ] + + report $"Discovered {newTests |> Array.sumBy (TestItem.runnableChildren >> Array.length)} tests" + rootTestCollection.replace (newTests |> ResizeArray) + + if builtTestProjects |> List.length > 0 && Array.length newTests = 0 then + let message = + "Detected test projects but no tests. Make sure your tests can be run with `dotnet test`" + + window.showWarningMessage (message) |> ignore + logger.Warn(message) + + else + let possibleDiscoveryFailures = + Array.concat + [ let getProjectTests (ti: TestItem) = ti.children.TestItems() + + listDiscoveredPerProject + |> Array.filter (snd >> getProjectTests >> Array.isEmpty) + |> Array.map (fst >> ProjectPath.fromProject) + + trxDiscoveryProjects + |> Array.ofList + |> Array.map ProjectPath.fromProject + |> Array.filter (makeTrxPath >> Path.tryPath >> Option.isNone) ] + + if (not << Array.isEmpty) possibleDiscoveryFailures then + let projectList = String.Join("\n", possibleDiscoveryFailures) + + warn + $"No tests discovered for the following projects. Make sure your tests can be run with `dotnet test` \n {projectList}" + } + let refreshTestList testItemFactory (rootTestCollection: TestItemCollection) tryGetLocation makeTrxPath + useLegacyDotnetCliIntegration (cancellationToken: CancellationToken) = @@ -1562,11 +1757,6 @@ module Interactions = {| message = Some message increment = None |} - let warn (message: string) = - logger.Warn(message) - window.showWarningMessage (message) |> ignore - - let cancellationToken = CancellationToken.mergeTokens [ cancellationToken; progressCancelToken ] @@ -1605,114 +1795,26 @@ module Interactions = window.showErrorMessage (message) |> ignore logger.Error(message, buildFailures |> List.map ProjectPath.fromProject) + else if useLegacyDotnetCliIntegration then + do! + discoverTestsWithDotnetCli + testItemFactory + tryGetLocation + makeTrxPath + report + rootTestCollection + cancellationToken + builtTestProjects else - let detectablePackageToFramework = - dict - [ "Expecto", TestFrameworkId.Expecto - "xunit.abstractions", TestFrameworkId.XUnit ] - - let librariesCapableOfListOnlyDiscovery = set detectablePackageToFramework.Keys - - let listDiscoveryProjects, trxDiscoveryProjects = - builtTestProjects - |> List.partition (fun project -> - project.PackageReferences - |> Array.exists (fun pr -> librariesCapableOfListOnlyDiscovery |> Set.contains pr.Name)) - - let discoverTestsByListOnly (project: Project) = - promise { - report $"Discovering tests for {project.Project}" - - let! testNames = - DotnetCli.listTests project.Project project.Info.TargetFramework false cancellationToken - - let detectedTestFramework = - let getPackageName (pr: PackageReference) = pr.Name - - project.PackageReferences - |> Array.tryPick (getPackageName >> Dict.tryGet detectablePackageToFramework) - - let testItemFactory (testItemBuilder: TestItem.TestItemBuilder) = - testItemFactory - { testItemBuilder with - testFramework = detectedTestFramework } - - let testHierarchy = - testNames - |> Array.map (fun n -> {| FullName = n; Data = () |}) - |> TestName.inferHierarchy - |> Array.map ( - TestItem.fromNamedHierarchy testItemFactory tryGetLocation project.Project - ) - - return - TestItem.fromProject - testItemFactory - project.Project - project.Info.TargetFramework - testHierarchy - } - - - let! listDiscoveredPerProject = - listDiscoveryProjects - |> ListExt.mapKeepInputAsync discoverTestsByListOnly - |> Promise.all - - trxDiscoveryProjects - |> List.iter (ProjectPath.fromProject >> makeTrxPath >> Path.deleteIfExists) - - let! _ = - trxDiscoveryProjects - |> Promise.executeWithMaxParallel maxParallelTestProjects (fun project -> - let projectPath = project.Project - report $"Discovering tests for {projectPath}" - let trxPath = makeTrxPath projectPath |> Some - - DotnetCli.test - projectPath - project.Info.TargetFramework - trxPath - None - DotnetCli.DebugTests.NoDebug - cancellationToken) - - let trxDiscoveredTests = - TestDiscovery.discoverFromTrx testItemFactory tryGetLocation makeTrxPath trxDiscoveryProjects - - - let listDiscoveredTests = listDiscoveredPerProject |> Array.map snd - let newTests = Array.concat [ listDiscoveredTests; trxDiscoveredTests ] - - report $"Discovered {newTests |> Array.sumBy (TestItem.runnableChildren >> Array.length)} tests" - rootTestCollection.replace (newTests |> ResizeArray) - - if testProjectCount > 0 && Array.length newTests = 0 then - let message = - "Detected test projects but no tests. Make sure your tests can be run with `dotnet test`" - - window.showWarningMessage (message) |> ignore - logger.Warn(message) - - else - let possibleDiscoveryFailures = - Array.concat - [ let getProjectTests (ti: TestItem) = ti.children.TestItems() - - listDiscoveredPerProject - |> Array.filter (snd >> getProjectTests >> Array.isEmpty) - |> Array.map (fst >> ProjectPath.fromProject) + let! discoveryResponse = LanguageService.testDiscovery () - trxDiscoveryProjects - |> Array.ofList - |> Array.map ProjectPath.fromProject - |> Array.filter (makeTrxPath >> Path.tryPath >> Option.isNone) ] + let testItems = + discoveryResponse.Data + |> TestItem.ofTestDTOs testItemFactory tryGetLocation + |> ResizeArray - if (not << Array.isEmpty) possibleDiscoveryFailures then - let projectList = String.Join("\n", possibleDiscoveryFailures) + rootTestCollection.replace (testItems) - warn - $"No tests discovered for the following projects. Make sure your tests can be run with `dotnet test` \n {projectList}" } let tryMatchTestBySuffix (locationCache: CodeLocationCache) (testId: TestId) = @@ -1796,73 +1898,6 @@ module Mailbox = idleLoop () -module TestItemDTO = - - let testItemsOfTestDTOs testItemFactory tryGetLocation (flatTests: TestItemDTO array) = - - let fromTestItemDTO - (constructId: FullTestName -> TestId) - (itemFactory: TestItem.TestItemFactory) - (tryGetLocation: TestId -> LocationRecord option) - (hierarchy: TestName.NameHierarchy) - : TestItem = - let rec recurse (namedNode: TestName.NameHierarchy) = - let id = constructId namedNode.FullName - - let toUri path = - try - if String.IsNullOrEmpty path then - None - else - vscode.Uri.parse ($"file:///{path}", true) |> Some - with e -> - logger.Debug($"Failed to parse test location uri {path}", e) - None - - let toRange (rangeDto: TestFileRange) = - vscode.Range.Create( - vscode.Position.Create(rangeDto.StartLine, 0), - vscode.Position.Create(rangeDto.EndLine, 0) - ) - // TODO: figure out how to incorporate cached location data - itemFactory - { id = id - label = namedNode.Name - uri = namedNode.Data |> Option.bind (fun t -> t.CodeFilePath) |> Option.bind toUri - range = - namedNode.Data - |> Option.bind (fun t -> t.CodeLocationRange) - |> Option.map toRange - // uri = location |> LocationRecord.tryGetUri - // range = location |> LocationRecord.tryGetRange - children = namedNode.Children |> Array.map recurse - testFramework = - namedNode.Data - |> Option.bind (fun t -> t.ExecutorUri |> TrxParser.adapterTypeNameToTestFramework) } - - recurse hierarchy - - - let mapDtosForProject ((projectPath, targetFramework), flatTests) = - let namedHierarchies = - flatTests - |> Array.map (fun t -> {| Data = t; FullName = t.FullName |}) - |> TestName.inferHierarchy - - let projectChildTestItems = - namedHierarchies - |> Array.map (fromTestItemDTO (TestItem.constructId projectPath) testItemFactory tryGetLocation) - - TestItem.fromProject testItemFactory projectPath targetFramework projectChildTestItems - - let testDtosByProject = - flatTests |> Array.groupBy (fun dto -> dto.ProjectFilePath, dto.TargetFramework) - - let testItemsByProject = testDtosByProject |> Array.map mapDtosForProject - - testItemsByProject - - let activate (context: ExtensionContext) = let useLegacyDotnetCliIntegration = @@ -1920,31 +1955,22 @@ let activate (context: ExtensionContext) = let refreshHandler cancellationToken = - if useLegacyDotnetCliIntegration then - Interactions.refreshTestList - testItemFactory - testController.items - tryGetLocation - makeTrxPath - cancellationToken - |> Promise.toThenable - |> (!^) - else - promise { - try - let! discoveryResponse = LanguageService.testDiscovery () - - let testItems = - discoveryResponse.Data - |> TestItemDTO.testItemsOfTestDTOs testItemFactory tryGetLocation - |> ResizeArray + promise { + try + do! + Interactions.refreshTestList + testItemFactory + testController.items + tryGetLocation + makeTrxPath + useLegacyDotnetCliIntegration + cancellationToken + with e -> + logger.Error("Ionide test discovery threw an exception", e) + } + |> Promise.toThenable + |> (!^) - testController.items.replace (testItems) - with e -> - logger.Error("Ionide test discovery threw an exception", e) - } - |> Promise.toThenable - |> (!^) testController.refreshHandler <- Some refreshHandler @@ -1957,12 +1983,13 @@ let activate (context: ExtensionContext) = if shouldAutoDiscoverTests && not hasInitiatedDiscovery then hasInitiatedDiscovery <- true - let trxTests = - TestDiscovery.discoverFromTrx testItemFactory tryGetLocation makeTrxPath + if useLegacyDotnetCliIntegration then + let trxTests = + TestDiscovery.discoverFromTrx testItemFactory tryGetLocation makeTrxPath - let workspaceProjects = Project.getLoaded () - let initialTests = trxTests workspaceProjects - initialTests |> Array.iter testController.items.add + let workspaceProjects = Project.getLoaded () + let initialTests = trxTests workspaceProjects + initialTests |> Array.iter testController.items.add let cancellationTokenSource = vscode.CancellationTokenSource.Create() // NOTE: Trx results can be partial if the last test run was filtered, so also queue a refresh to make sure we discover all tests @@ -1971,6 +1998,7 @@ let activate (context: ExtensionContext) = testController.items tryGetLocation makeTrxPath + useLegacyDotnetCliIntegration cancellationTokenSource.token |> Promise.start diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index 2b250fef..f37fef62 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -597,12 +597,10 @@ Consider: let testDiscovery s = match client with - | None -> - Promise.empty + | None -> Promise.empty | Some cl -> cl.sendRequest ("test/discoverTests", ()) - |> Promise.map (fun (res: Types.PlainNotification) -> - res.content |> ofJson) + |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) let private createClient (opts: Executable) = From b5435e9d8c17d0cdcddf166d64ee26e3d180503f Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Fri, 25 Jul 2025 17:17:45 -0500 Subject: [PATCH 03/32] Primative merge of code locations on test discovery --- src/Components/TestExplorer.fs | 72 ++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index ad23b061..25bdad5d 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -679,7 +679,16 @@ type CodeLocationCache() = locationCache.Remove(kvp.Key) |> ignore +module Option = + let tee (f: 'a -> unit) (option: 'a option) = + option |> Option.iter f + option + + let tryFallback f opt = + match opt with + | Some _ -> opt + | None -> f () module TestItem = @@ -857,35 +866,43 @@ module TestItem = (tryGetLocation: TestId -> LocationRecord option) (hierarchy: TestName.NameHierarchy) : TestItem = + let toUri path = + try + if String.IsNullOrEmpty path then + None + else + vscode.Uri.parse ($"file:///{path}", true) |> Some + with e -> + logger.Debug($"Failed to parse test location uri {path}", e) + None + + let toRange (rangeDto: TestFileRange) = + vscode.Range.Create( + vscode.Position.Create(rangeDto.StartLine, 0), + vscode.Position.Create(rangeDto.EndLine, 0) + ) + + let tryDtoToLocation (dto: TestItemDTO) : LocationRecord option = + match dto.CodeFilePath |> Option.bind toUri, dto.CodeLocationRange with + | Some path, Some range -> + { Uri = path + Range = toRange (range) |> Some } + |> Some + | _ -> None + let rec recurse (namedNode: TestName.NameHierarchy) = let id = constructId namedNode.FullName - let toUri path = - try - if String.IsNullOrEmpty path then - None - else - vscode.Uri.parse ($"file:///{path}", true) |> Some - with e -> - logger.Debug($"Failed to parse test location uri {path}", e) - None + let codeLocation = + namedNode.Data + |> Option.bind tryDtoToLocation + |> Option.tryFallback (fun _ -> tryGetLocation id) - let toRange (rangeDto: TestFileRange) = - vscode.Range.Create( - vscode.Position.Create(rangeDto.StartLine, 0), - vscode.Position.Create(rangeDto.EndLine, 0) - ) - // TODO: figure out how to incorporate cached location data itemFactory { id = id label = namedNode.Name - uri = namedNode.Data |> Option.bind (fun t -> t.CodeFilePath) |> Option.bind toUri - range = - namedNode.Data - |> Option.bind (fun t -> t.CodeLocationRange) - |> Option.map toRange - // uri = location |> LocationRecord.tryGetUri - // range = location |> LocationRecord.tryGetRange + uri = codeLocation |> LocationRecord.tryGetUri + range = codeLocation |> LocationRecord.tryGetRange children = namedNode.Children |> Array.map recurse testFramework = namedNode.Data @@ -1823,17 +1840,6 @@ module Interactions = locationCache.GetKnownTestIds() |> Seq.tryFind (matcher testId) - module Option = - - let tee (f: 'a -> unit) (option: 'a option) = - option |> Option.iter f - option - - let tryFallback f opt = - match opt with - | Some _ -> opt - | None -> f () - let tryGetLocation (locationCache: CodeLocationCache) testId = let cached = locationCache.GetById testId From baad24113e2bfa6305cd809de11245f9c8e6ee20 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:09:16 -0500 Subject: [PATCH 04/32] Implement incremental test discovery updates --- src/Components/TestExplorer.fs | 42 ++++++++++++++++++++++++++++------ src/Core/DTO.fs | 3 +++ src/Core/LanguageService.fs | 9 +++++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 25bdad5d..b59ac314 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -910,7 +910,6 @@ module TestItem = recurse hierarchy - let mapDtosForProject ((projectPath, targetFramework), flatTests) = let namedHierarchies = flatTests @@ -1351,14 +1350,14 @@ module Interactions = // Potentially we are going to run multiple tests that match this filter let testPart = escapedTestName.Split(' ').[0] $"(FullyQualifiedName~{testPart})" + // NOTE: using DisplayName allows single theory cases to be run for xUnit else if test.TestFramework = TestFrameworkId.XUnit then - // NOTE: using DisplayName allows single theory cases to be run for xUnit let operator = if test.children.size = 0 then "=" else "~" $"(DisplayName{operator}{escapedTestName})" + // NOTE: MSTest can't filter to parameterized test cases + // Truncating before the case parameters will run all the theory cases + // example parameterized test name -> `MsTestTests.TestClass.theoryTest (2,3,5)` else if test.TestFramework = TestFrameworkId.MsTest && String.endWith ")" fullTestName then - // NOTE: MSTest can't filter to parameterized test cases - // Truncating before the case parameters will run all the theory cases - // example parameterized test name -> `MsTestTests.TestClass.theoryTest (2,3,5)` let truncateOnLast (separator: string) (toSplit: string) = match toSplit.LastIndexOf(separator) with | -1 -> toSplit @@ -1823,7 +1822,37 @@ module Interactions = cancellationToken builtTestProjects else - let! discoveryResponse = LanguageService.testDiscovery () + let mergeTestItemCollections (target: TestItem array) (addition: TestItem array) : TestItem array = + let rec recurse (target: TestItem array) (addition: TestItem array) : TestItem array = + let targetOnly, conficted, addedOnly = + ArrayExt.venn TestItem.getId TestItem.getId target addition + + let mergeSingle (targetItem: TestItem, addedItem: TestItem) = + let mergedChildren = + recurse (targetItem.children.TestItems()) (addedItem.children.TestItems()) + + addedItem.children.replace (ResizeArray mergedChildren) + addedItem + + Array.concat [ targetOnly; addedOnly; conficted |> Array.map mergeSingle ] + + recurse target addition + + let mutable discoveredTestsAccumulator: TestItem array = + rootTestCollection.TestItems() + + let incrementalUpdateHandler (discoveryUpdate: TestDiscoveryUpdate) : unit = + try + let newItems = + discoveryUpdate.Tests |> TestItem.ofTestDTOs testItemFactory tryGetLocation + + discoveredTestsAccumulator <- mergeTestItemCollections discoveredTestsAccumulator newItems + + rootTestCollection.replace (ResizeArray discoveredTestsAccumulator) + with e -> + logger.Debug("Incremental test discovery update threw an exception", e) + + let! discoveryResponse = LanguageService.testDiscovery incrementalUpdateHandler () let testItems = discoveryResponse.Data @@ -1831,7 +1860,6 @@ module Interactions = |> ResizeArray rootTestCollection.replace (testItems) - } let tryMatchTestBySuffix (locationCache: CodeLocationCache) (testId: TestId) = diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index c9216f28..8c40a7fe 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -384,6 +384,9 @@ module DTO = CodeLocationRange: TestFileRange option } + type TestDiscoveryUpdate = { Tests: TestItemDTO array } + + type Result<'T> = { Kind: string; Data: 'T } type HelptextResult = Result diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index f37fef62..eb416a1f 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -595,10 +595,17 @@ Consider: let fsiSdk () = promise { return Environment.configFsiSdkFilePath () } - let testDiscovery s = + let testDiscovery incrementalUpdateHandler () = match client with | None -> Promise.empty | Some cl -> + cl.onNotification ( + "test/testDiscoveryUpdate", + (fun (notification: Types.PlainNotification) -> + let parsed = ofJson notification.content + incrementalUpdateHandler parsed) + ) + cl.sendRequest ("test/discoverTests", ()) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) From a713c9954466548da05aefcb6faa46dbdabb74ed Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 30 Jul 2025 15:33:26 -0500 Subject: [PATCH 05/32] Run all tests using the language server Does not currently work for partial test runs. Also doesn't work if there are MSTest theory tests due to discrepancy between test discovery and test results --- src/Components/TestExplorer.fs | 169 ++++++++++++++++++++++++--------- src/Core/DTO.fs | 17 ++++ src/Core/LanguageService.fs | 9 +- 3 files changed, 149 insertions(+), 46 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index b59ac314..5ed420ee 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -79,6 +79,22 @@ module Dict = let tryGet (d: Collections.Generic.IDictionary<'key, 'value>) (key) : 'value option = if d.ContainsKey(key) then Some d[key] else None +module Option = + + let tee (f: 'a -> unit) (option: 'a option) = + option |> Option.iter f + option + + let tryFallback f opt = + match opt with + | Some _ -> opt + | None -> f () + + let tryFallbackValue fallbackOpt opt = + match opt with + | Some _ -> opt + | None -> fallbackOpt + module CancellationToken = let mergeTokens (tokens: CancellationToken list) = let tokenSource = vscode.CancellationTokenSource.Create() @@ -220,6 +236,16 @@ type TestResultOutcome = | NotExecuted | Failed | Passed + | Skipped + +module TestResultOutcome = + let ofOutcomeDto (outcomeDto: TestOutcomeDTO) = + match outcomeDto with + | TestOutcomeDTO.Failed -> TestResultOutcome.Failed + | TestOutcomeDTO.Passed -> TestResultOutcome.Passed + | TestOutcomeDTO.Skipped -> TestResultOutcome.Skipped + | TestOutcomeDTO.None -> TestResultOutcome.NotExecuted + | TestOutcomeDTO.NotFound -> TestResultOutcome.NotExecuted type TestFrameworkId = string @@ -237,6 +263,18 @@ module TestFrameworkId = [] let Expecto = "Expecto" + let tryFromExecutorUri adapterTypeName = + if String.startWith "executor://nunit" adapterTypeName then + Some NUnit + else if String.startWith "executor://mstest" adapterTypeName then + Some MsTest + else if String.startWith "executor://xunit" adapterTypeName then + Some XUnit + else if String.startWith "executor://yolodev" adapterTypeName then + Some Expecto + else + None + type TestResult = { FullTestName: string Outcome: TestResultOutcome @@ -248,6 +286,37 @@ type TestResult = Timing: float TestFramework: TestFrameworkId option } +module TestResult = + let tryExtractExpectedAndActual (message: string option) = + let expected, actual = + match message with + | None -> None, None + | Some message -> + let lines = + message.Split([| "\r\n"; "\n" |], StringSplitOptions.RemoveEmptyEntries) + |> Array.map (fun n -> n.TrimStart()) + + let tryFind (startsWith: string) = + Array.tryFind (fun (line: string) -> line.StartsWith(startsWith)) lines + |> Option.map (fun line -> line.Replace(startsWith, "").TrimStart()) + + tryFind "Expected:", tryFind "But was:" |> Option.tryFallbackValue (tryFind "Actual:") + + expected, actual + + let ofTestResultDTO (testResultDto: TestResultDTO) = + let expected, actual = tryExtractExpectedAndActual testResultDto.ErrorMessage + + { FullTestName = testResultDto.TestItem.FullName + Outcome = testResultDto.Outcome |> TestResultOutcome.ofOutcomeDto + Output = testResultDto.AdditionalOutput + ErrorMessage = testResultDto.ErrorMessage + ErrorStackTrace = testResultDto.ErrorStackTrace + Timing = testResultDto.Duration.Milliseconds + TestFramework = testResultDto.TestItem.ExecutorUri |> TestFrameworkId.tryFromExecutorUri + Expected = expected + Actual = actual } + module Path = let tryPath (path: string) = @@ -276,18 +345,6 @@ module Path = module TrxParser = - let adapterTypeNameToTestFramework adapterTypeName = - if String.startWith "executor://nunit" adapterTypeName then - Some TestFrameworkId.NUnit - else if String.startWith "executor://mstest" adapterTypeName then - Some TestFrameworkId.MsTest - else if String.startWith "executor://xunit" adapterTypeName then - Some TestFrameworkId.XUnit - else if String.startWith "executor://yolodev" adapterTypeName then - Some TestFrameworkId.Expecto - else - None - type Execution = { Id: string } type TestMethod = @@ -304,7 +361,7 @@ module TrxParser = // IMPORTANT: XUnit and MSTest don't include the parameterized test case data in the TestMethod.Name // but NUnit and MSTest don't use fully qualified names in UnitTest.Name. // Therefore, we have to conditionally build this full name based on the framework - match self.TestMethod.AdapterTypeName |> adapterTypeNameToTestFramework with + match self.TestMethod.AdapterTypeName |> TestFrameworkId.tryFromExecutorUri with | Some TestFrameworkId.NUnit -> TestName.fromPathAndTestName self.TestMethod.ClassName self.TestMethod.Name | Some TestFrameworkId.MsTest -> TestName.fromPathAndTestName self.TestMethod.ClassName self.Name | _ -> self.Name @@ -679,17 +736,6 @@ type CodeLocationCache() = locationCache.Remove(kvp.Key) |> ignore -module Option = - - let tee (f: 'a -> unit) (option: 'a option) = - option |> Option.iter f - option - - let tryFallback f opt = - match opt with - | Some _ -> opt - | None -> f () - module TestItem = let private idSeparator = " -- " @@ -906,7 +952,7 @@ module TestItem = children = namedNode.Children |> Array.map recurse testFramework = namedNode.Data - |> Option.bind (fun t -> t.ExecutorUri |> TrxParser.adapterTypeNameToTestFramework) } + |> Option.bind (fun t -> t.ExecutorUri |> TestFrameworkId.tryFromExecutorUri) } recurse hierarchy @@ -1193,7 +1239,7 @@ module TestDiscovery = (fun nh -> nh.Data |> Option.bind (fun (trxDef: TrxParser.UnitTest) -> - TrxParser.adapterTypeNameToTestFramework trxDef.TestMethod.AdapterTypeName)) + TestFrameworkId.tryFromExecutorUri trxDef.TestMethod.AdapterTypeName)) hierarchy let testItemFactory (testItemBuilder: TestItem.TestItemBuilder) = @@ -1379,6 +1425,7 @@ module Interactions = match testResult.Outcome with | TestResultOutcome.NotExecuted -> testRun.skipped testItem + | TestResultOutcome.Skipped -> testRun.skipped testItem | TestResultOutcome.Passed -> testResult.Output |> Option.iter (TestRun.appendOutputLineForTest testRun testItem) @@ -1450,19 +1497,9 @@ module Interactions = displayTestResultInExplorer testRun (treeItem, additionalResult)) let private trxResultToTestResult (trxResult: TrxParser.TestWithResult) = - let expected, actual = - match trxResult.UnitTestResult.Output.ErrorInfo.Message with - | None -> None, None - | Some message -> - let lines = - message.Split([| "\r\n"; "\n" |], StringSplitOptions.RemoveEmptyEntries) - |> Array.map (fun n -> n.TrimStart()) - - let tryFind (startsWith: string) = - Array.tryFind (fun (line: string) -> line.StartsWith(startsWith)) lines - |> Option.map (fun line -> line.Replace(startsWith, "").TrimStart()) - tryFind "Expected:", tryFind "But was:" + let expected, actual = + TestResult.tryExtractExpectedAndActual trxResult.UnitTestResult.Output.ErrorInfo.Message { FullTestName = trxResult.UnitTest.FullName Outcome = !!trxResult.UnitTestResult.Outcome @@ -1472,7 +1509,7 @@ module Interactions = Expected = expected Actual = actual Timing = trxResult.UnitTestResult.Duration.Milliseconds - TestFramework = TrxParser.adapterTypeNameToTestFramework trxResult.UnitTest.TestMethod.AdapterTypeName } + TestFramework = TestFrameworkId.tryFromExecutorUri trxResult.UnitTest.TestMethod.AdapterTypeName } type MergeTestResultsToExplorer = TestRun -> ProjectPath -> TargetFramework -> TestItem array -> TestResult array -> unit @@ -1554,7 +1591,10 @@ module Interactions = - let private filtersToProjectRunRequests (rootTestCollection: TestItemCollection) (runRequest: TestRunRequest) = + let private filtersToProjectRunRequests + (rootTestCollection: TestItemCollection) + (runRequest: TestRunRequest) + : ProjectRunRequest array = let testSelection = runRequest.``include`` |> Option.map Array.ofSeq @@ -1612,6 +1652,7 @@ module Interactions = (testController: TestController) (tryGetLocation: TestId -> LocationRecord option) (makeTrxPath) + (useLegacyDotnetCliIntegration: bool) (req: TestRunRequest) (_ct: CancellationToken) : U2, unit> = @@ -1664,9 +1705,46 @@ module Interactions = let successfullyBuiltRequests = buildResults |> List.choose id - let! _ = - successfullyBuiltRequests - |> (Promise.executeWithMaxParallel maxParallelTestProjects runTestProject) + if useLegacyDotnetCliIntegration then + let! _ = + successfullyBuiltRequests + |> (Promise.executeWithMaxParallel maxParallelTestProjects runTestProject) + + () + else + try + let runRequestDictionary = + projectRunRequests |> Array.map (fun rr -> rr.ProjectPath, rr) |> Map + + logger.Debug("Nya: made runRequestDictionary") + let! runResult = LanguageService.runTests () + logger.Debug("Nya: server test run complete", runResult) + + let groups = + runResult.Data + |> Array.groupBy (fun (tr: TestResultDTO) -> + tr.TestItem.ProjectFilePath, tr.TestItem.TargetFramework) + + logger.Debug("Nya: results grouped", runResult) + + groups + |> Array.iter (fun ((projPath, targetFramework), results) -> + logger.Debug("Nya: merging for project", projPath, results) + + let expectedToRun = + runRequestDictionary + |> Map.tryFind projPath + |> Option.map (fun rr -> rr.Tests |> Array.collect TestItem.runnableChildren) + |> Option.defaultValue Array.empty + + logger.Debug("Nya: expected to run", expectedToRun) + + + let actuallyRan = results |> Array.map TestResult.ofTestResultDTO + logger.Debug("Nya: actuallyRan", actuallyRan) + mergeTestResultsToExplorer testRun projPath targetFramework expectedToRun actuallyRan) + with ex -> + logger.Debug("Nya: test run failed with exception", ex) testRun.``end`` () } @@ -1852,7 +1930,7 @@ module Interactions = with e -> logger.Debug("Incremental test discovery update threw an exception", e) - let! discoveryResponse = LanguageService.testDiscovery incrementalUpdateHandler () + let! discoveryResponse = LanguageService.discoverTests incrementalUpdateHandler () let testItems = discoveryResponse.Data @@ -1954,7 +2032,8 @@ let activate (context: ExtensionContext) = let tryGetLocation = Interactions.tryGetLocation locationCache - let runHandler = Interactions.runHandler testController tryGetLocation makeTrxPath + let runHandler = + Interactions.runHandler testController tryGetLocation makeTrxPath useLegacyDotnetCliIntegration testController.createRunProfile ("Run F# Tests", TestRunProfileKind.Run, runHandler, true) |> unbox diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index 8c40a7fe..e97d030d 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -384,6 +384,22 @@ module DTO = CodeLocationRange: TestFileRange option } + [] + type TestOutcomeDTO = + | Failed = 0 + | Passed = 1 + | Skipped = 2 + | None = 3 + | NotFound = 4 + + type TestResultDTO = + { TestItem: TestItemDTO + Outcome: TestOutcomeDTO + ErrorMessage: string option + ErrorStackTrace: string option + AdditionalOutput: string option + Duration: System.TimeSpan } + type TestDiscoveryUpdate = { Tests: TestItemDTO array } @@ -416,6 +432,7 @@ module DTO = type PipelineHintsResult = Result type TestResult = Result type DiscoverTestsResult = Result + type RunTestsResult = Result module DotnetNew = diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index eb416a1f..0afc37b0 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -595,7 +595,7 @@ Consider: let fsiSdk () = promise { return Environment.configFsiSdkFilePath () } - let testDiscovery incrementalUpdateHandler () = + let discoverTests incrementalUpdateHandler () = match client with | None -> Promise.empty | Some cl -> @@ -609,6 +609,13 @@ Consider: cl.sendRequest ("test/discoverTests", ()) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) + let runTests () = + match client with + | None -> Promise.empty + | Some cl -> + cl.sendRequest ("test/runTests", ()) + |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) + let private createClient (opts: Executable) = let options: ServerOptions = U5.Case2 {| run = opts; debug = opts |} From fd860a6bf16c5a57bdf43bbe67c43f40a7266a0d Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Mon, 4 Aug 2025 12:50:48 -0500 Subject: [PATCH 06/32] Improve discovery progress reporting --- src/Components/TestExplorer.fs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 5ed420ee..11498c7a 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1919,17 +1919,22 @@ module Interactions = let mutable discoveredTestsAccumulator: TestItem array = rootTestCollection.TestItems() + let mutable discoveredTestCount: int = 0 + let incrementalUpdateHandler (discoveryUpdate: TestDiscoveryUpdate) : unit = try let newItems = discoveryUpdate.Tests |> TestItem.ofTestDTOs testItemFactory tryGetLocation discoveredTestsAccumulator <- mergeTestItemCollections discoveredTestsAccumulator newItems + discoveredTestCount <- discoveredTestCount + (discoveryUpdate.Tests |> Array.length) + report $"Discovering tests: {discoveredTestCount} discovered" rootTestCollection.replace (ResizeArray discoveredTestsAccumulator) with e -> logger.Debug("Incremental test discovery update threw an exception", e) + report "Discovering tests" let! discoveryResponse = LanguageService.discoverTests incrementalUpdateHandler () let testItems = From bbb6a1727059f763b8394fca7e3b33f238d9fa75 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:36:52 -0500 Subject: [PATCH 07/32] Fix bug when merging theory test results to the explorer --- src/Components/TestExplorer.fs | 38 ++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 11498c7a..9f520e79 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -275,6 +275,21 @@ module TestFrameworkId = else None +module TestItemDTO = + let getNormalizedFullName (dto: TestItemDTO) = + match dto.ExecutorUri |> TestFrameworkId.tryFromExecutorUri with + // NOTE: XUnit and MSTest don't include the theory case parameters in the FullyQualifiedName, but do include them in the DisplayName. + // Thus we need to append the DisplayName to differentiate the test cases + | Some TestFrameworkId.MsTest -> + if dto.FullName.EndsWith(dto.DisplayName) then + dto.FullName + else + dto.FullName + "." + dto.DisplayName + | Some TestFrameworkId.XUnit -> + // NOTE: XUnit includes the FullyQualifiedName in the DisplayName + dto.DisplayName + | _ -> dto.FullName + type TestResult = { FullTestName: string Outcome: TestResultOutcome @@ -307,7 +322,7 @@ module TestResult = let ofTestResultDTO (testResultDto: TestResultDTO) = let expected, actual = tryExtractExpectedAndActual testResultDto.ErrorMessage - { FullTestName = testResultDto.TestItem.FullName + { FullTestName = testResultDto.TestItem |> TestItemDTO.getNormalizedFullName Outcome = testResultDto.Outcome |> TestResultOutcome.ofOutcomeDto Output = testResultDto.AdditionalOutput ErrorMessage = testResultDto.ErrorMessage @@ -957,10 +972,12 @@ module TestItem = recurse hierarchy let mapDtosForProject ((projectPath, targetFramework), flatTests) = + let testDtoToNamedItem (dto: TestItemDTO) = + {| Data = dto + FullName = dto |> TestItemDTO.getNormalizedFullName |} + let namedHierarchies = - flatTests - |> Array.map (fun t -> {| Data = t; FullName = t.FullName |}) - |> TestName.inferHierarchy + flatTests |> Array.map testDtoToNamedItem |> TestName.inferHierarchy let projectChildTestItems = namedHierarchies @@ -1458,6 +1475,7 @@ module Interactions = (expectedToRun: TestItem array) (testResults: TestResult array) = + let tryRemove (testWithoutResult: TestItem) = let parentCollection = match testWithoutResult.parent with @@ -1716,35 +1734,25 @@ module Interactions = let runRequestDictionary = projectRunRequests |> Array.map (fun rr -> rr.ProjectPath, rr) |> Map - logger.Debug("Nya: made runRequestDictionary") let! runResult = LanguageService.runTests () - logger.Debug("Nya: server test run complete", runResult) let groups = runResult.Data |> Array.groupBy (fun (tr: TestResultDTO) -> tr.TestItem.ProjectFilePath, tr.TestItem.TargetFramework) - logger.Debug("Nya: results grouped", runResult) - groups |> Array.iter (fun ((projPath, targetFramework), results) -> - logger.Debug("Nya: merging for project", projPath, results) - let expectedToRun = runRequestDictionary |> Map.tryFind projPath |> Option.map (fun rr -> rr.Tests |> Array.collect TestItem.runnableChildren) |> Option.defaultValue Array.empty - logger.Debug("Nya: expected to run", expectedToRun) - - let actuallyRan = results |> Array.map TestResult.ofTestResultDTO - logger.Debug("Nya: actuallyRan", actuallyRan) mergeTestResultsToExplorer testRun projPath targetFramework expectedToRun actuallyRan) with ex -> - logger.Debug("Nya: test run failed with exception", ex) + logger.Debug("Test run failed with exception", ex) testRun.``end`` () } From 16527ce5f9dd9eb6501b7a381a9e5c1803bbe97e Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:42:00 -0500 Subject: [PATCH 08/32] Match Xunit theory case nesting to Visual Studio's --- src/Components/TestExplorer.fs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 9f520e79..04cefe7a 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -286,8 +286,13 @@ module TestItemDTO = else dto.FullName + "." + dto.DisplayName | Some TestFrameworkId.XUnit -> - // NOTE: XUnit includes the FullyQualifiedName in the DisplayName - dto.DisplayName + // NOTE: XUnit includes the FullyQualifiedName in the DisplayName. + // But it doesn't nest theory cases, just appends the case parameters + if dto.DisplayName <> dto.FullName then + let theoryCaseFragment = dto.DisplayName.Split('.') |> Array.last + dto.FullName + "." + theoryCaseFragment + else + dto.FullName | _ -> dto.FullName type TestResult = From 7967cb8e3a1652517dbc59163e00672f1f7f313b Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Mon, 4 Aug 2025 17:50:50 -0500 Subject: [PATCH 09/32] Demonstrate basic streaming of test results from langauge server --- src/Components/TestExplorer.fs | 62 ++++++++++++++++++++++++---------- src/Core/DTO.fs | 4 +++ src/Core/LanguageService.fs | 9 ++++- 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 04cefe7a..c8bbc4ea 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1477,6 +1477,7 @@ module Interactions = (testRun: TestRun) (projectPath: ProjectPath) (targetFramework: TargetFramework) + (shouldDeleteMissing: bool) (expectedToRun: TestItem array) (testResults: TestResult array) = @@ -1510,7 +1511,9 @@ module Interactions = ArrayExt.venn treeItemComparable resultComparable expectedToRun testResults expected |> Array.iter (displayTestResultInExplorer testRun) - missing |> Array.iter tryRemove + + if shouldDeleteMissing then + missing |> Array.iter tryRemove added |> Array.iter (fun additionalResult -> @@ -1534,8 +1537,14 @@ module Interactions = Timing = trxResult.UnitTestResult.Duration.Milliseconds TestFramework = TestFrameworkId.tryFromExecutorUri trxResult.UnitTest.TestMethod.AdapterTypeName } + type TrimMissing = bool + + module TrimMissing = + let Trim = true + let NoTrim = false + type MergeTestResultsToExplorer = - TestRun -> ProjectPath -> TargetFramework -> TestItem array -> TestResult array -> unit + TestRun -> ProjectPath -> TargetFramework -> TrimMissing -> TestItem array -> TestResult array -> unit let private runTestProject_withoutExceptionHandling (mergeResultsToExplorer: MergeTestResultsToExplorer) @@ -1585,7 +1594,13 @@ module Interactions = window.showWarningMessage (message) |> ignore TestRun.appendOutputLine testRun message else - mergeResultsToExplorer testRun projectPath projectRunRequest.TargetFramework runnableTests testResults + mergeResultsToExplorer + testRun + projectPath + projectRunRequest.TargetFramework + TrimMissing.Trim + runnableTests + testResults } let runTestProject @@ -1739,23 +1754,36 @@ module Interactions = let runRequestDictionary = projectRunRequests |> Array.map (fun rr -> rr.ProjectPath, rr) |> Map - let! runResult = LanguageService.runTests () + let mergeResults (shouldTrim: TrimMissing) (resultDtos: TestResultDTO array) = + let groups = + resultDtos + |> Array.groupBy (fun tr -> tr.TestItem.ProjectFilePath, tr.TestItem.TargetFramework) + + groups + |> Array.iter (fun ((projPath, targetFramework), results) -> + let expectedToRun = + runRequestDictionary + |> Map.tryFind projPath + |> Option.map (fun rr -> rr.Tests |> Array.collect TestItem.runnableChildren) + |> Option.defaultValue Array.empty + + let actuallyRan: TestResult array = results |> Array.map TestResult.ofTestResultDTO + + mergeTestResultsToExplorer + testRun + projPath + targetFramework + shouldTrim + expectedToRun + actuallyRan) + + let incrementalUpdateHandler (runUpdate: TestRunUpdate) = + mergeResults TrimMissing.NoTrim runUpdate.TestResults - let groups = - runResult.Data - |> Array.groupBy (fun (tr: TestResultDTO) -> - tr.TestItem.ProjectFilePath, tr.TestItem.TargetFramework) + let! runResult = LanguageService.runTests incrementalUpdateHandler () - groups - |> Array.iter (fun ((projPath, targetFramework), results) -> - let expectedToRun = - runRequestDictionary - |> Map.tryFind projPath - |> Option.map (fun rr -> rr.Tests |> Array.collect TestItem.runnableChildren) - |> Option.defaultValue Array.empty + mergeResults TrimMissing.Trim runResult.Data - let actuallyRan = results |> Array.map TestResult.ofTestResultDTO - mergeTestResultsToExplorer testRun projPath targetFramework expectedToRun actuallyRan) with ex -> logger.Debug("Test run failed with exception", ex) diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index e97d030d..f716a0f3 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -402,6 +402,10 @@ module DTO = type TestDiscoveryUpdate = { Tests: TestItemDTO array } + type TestRunUpdate = + { TestResults: TestResultDTO array + ActiveTests: TestItemDTO array } + type Result<'T> = { Kind: string; Data: 'T } diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index 0afc37b0..ece4009c 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -609,10 +609,17 @@ Consider: cl.sendRequest ("test/discoverTests", ()) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) - let runTests () = + let runTests incrementalUpdateHandler () = match client with | None -> Promise.empty | Some cl -> + cl.onNotification ( + "test/testRunUpdate", + (fun (notification: Types.PlainNotification) -> + let parsed = ofJson notification.content + incrementalUpdateHandler parsed) + ) + cl.sendRequest ("test/runTests", ()) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) From 48e90eda27f31aa9b29b0cbd8c1e823dab6f7ccb Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Mon, 4 Aug 2025 19:09:35 -0500 Subject: [PATCH 10/32] Add incremental test running indicators to language server test runs --- src/Components/TestExplorer.fs | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index c8bbc4ea..5d74488a 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1751,8 +1751,16 @@ module Interactions = () else try - let runRequestDictionary = - projectRunRequests |> Array.map (fun rr -> rr.ProjectPath, rr) |> Map + let runnableTestsByProject = + projectRunRequests + |> Array.map (fun rr -> rr.ProjectPath, rr.Tests |> Array.collect TestItem.runnableChildren) + |> Map + + let expectedTestsById = + runnableTestsByProject + |> Map.values + |> Seq.collect (Seq.map (fun t -> t.id, t)) + |> Map let mergeResults (shouldTrim: TrimMissing) (resultDtos: TestResultDTO array) = let groups = @@ -1762,9 +1770,8 @@ module Interactions = groups |> Array.iter (fun ((projPath, targetFramework), results) -> let expectedToRun = - runRequestDictionary + runnableTestsByProject |> Map.tryFind projPath - |> Option.map (fun rr -> rr.Tests |> Array.collect TestItem.runnableChildren) |> Option.defaultValue Array.empty let actuallyRan: TestResult array = results |> Array.map TestResult.ofTestResultDTO @@ -1777,7 +1784,22 @@ module Interactions = expectedToRun actuallyRan) + let showStarted (testItems: TestItemDTO array) = + try + let groups = testItems |> Array.groupBy (fun t -> t.ProjectFilePath) + + groups + |> Array.iter (fun (projPath, activeTests) -> + let testIdsToStart = + activeTests |> Array.map (fun t -> TestItem.constructId projPath t.FullName) + + let knownExplorerItems = testIdsToStart |> Array.choose expectedTestsById.TryFind + knownExplorerItems |> TestRun.showStarted testRun) + with ex -> + logger.Debug("Threw error while mapping active test items to the explorer", ex) + let incrementalUpdateHandler (runUpdate: TestRunUpdate) = + showStarted runUpdate.ActiveTests mergeResults TrimMissing.NoTrim runUpdate.TestResults let! runResult = LanguageService.runTests incrementalUpdateHandler () From b04f7ca0aabd28089be4f68616f258b2c25386e9 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 5 Aug 2025 10:57:16 -0500 Subject: [PATCH 11/32] Add testCaseFilter support to test Runs Still has a bug where NUnit doesn't respect the filter, but works for all other frameworks --- src/Components/TestExplorer.fs | 14 +++++++++++++- src/Core/LanguageService.fs | 8 ++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 5d74488a..0b48a3bc 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1802,7 +1802,19 @@ module Interactions = showStarted runUpdate.ActiveTests mergeResults TrimMissing.NoTrim runUpdate.TestResults - let! runResult = LanguageService.runTests incrementalUpdateHandler () + let filterExpression = + match req.``include`` with + | None -> None + | Some selectedCases when Seq.isEmpty selectedCases -> None + | Some selectedCases -> + projectRunRequests + |> Array.collect (fun rr -> rr.Tests) + |> buildFilterExpression + |> Some + + logger.Debug($"Test Filter Expression: {filterExpression}") + + let! runResult = LanguageService.runTests incrementalUpdateHandler filterExpression mergeResults TrimMissing.Trim runResult.Data diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index ece4009c..c2a935e4 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -94,6 +94,8 @@ module LanguageService = { start: Fable.Import.VSCode.Vscode.Position ``end``: Fable.Import.VSCode.Vscode.Position } + type TestRunRequest = { TestCaseFilter: string option } + type Uri with member uri.ToDocumentUri = uri.ToString() @@ -609,7 +611,7 @@ Consider: cl.sendRequest ("test/discoverTests", ()) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) - let runTests incrementalUpdateHandler () = + let runTests incrementalUpdateHandler (testCaseFilter: string option) = match client with | None -> Promise.empty | Some cl -> @@ -620,7 +622,9 @@ Consider: incrementalUpdateHandler parsed) ) - cl.sendRequest ("test/runTests", ()) + let request: Types.TestRunRequest = { TestCaseFilter = testCaseFilter } + + cl.sendRequest ("test/runTests", request) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) let private createClient (opts: Executable) = From 2ff7a099578fb83c22e1a5fad3ec381ed08e098e Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:55:46 -0500 Subject: [PATCH 12/32] Draft: Attempt to attach debugger on test run. Currently hanging on the server side for unclear reasons --- src/Components/TestExplorer.fs | 79 +++++++++++++++++++--------------- src/Core/DTO.fs | 5 ++- src/Core/LanguageService.fs | 33 +++++++++++--- 3 files changed, 76 insertions(+), 41 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 0b48a3bc..d84b8c87 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -508,6 +508,30 @@ module TrxParser = |> TestName.inferHierarchy +module VSCodeActions = + let launchDebugger processId = + let launchRequest: DebugConfiguration = + {| name = ".NET Core Attach" + ``type`` = "coreclr" + request = "attach" + processId = processId |} + |> box + |> unbox + + let folder = workspace.workspaceFolders.Value.[0] + + promise { + let! _ = + Vscode.debug.startDebugging (Some folder, U2.Case2 launchRequest) + |> Promise.ofThenable + + // NOTE: Have to wait or it'll continue before the debugger reaches the stop on entry point. + // That'll leave the debugger in a confusing state where it shows it's attached but + // no breakpoints are hit and the breakpoints show as disabled + do! Promise.sleep 2000 + Vscode.commands.executeCommand ("workbench.action.debug.continue") |> ignore + } + |> ignore module DotnetCli = @@ -582,30 +606,6 @@ module DotnetCli = else None - let private launchDebugger processId = - let launchRequest: DebugConfiguration = - {| name = ".NET Core Attach" - ``type`` = "coreclr" - request = "attach" - processId = processId |} - |> box - |> unbox - - let folder = workspace.workspaceFolders.Value.[0] - - promise { - let! _ = - Vscode.debug.startDebugging (Some folder, U2.Case2 launchRequest) - |> Promise.ofThenable - - // NOTE: Have to wait or it'll continue before the debugger reaches the stop on entry point. - // That'll leave the debugger in a confusing state where it shows it's attached but - // no breakpoints are hit and the breakpoints show as disabled - do! Promise.sleep 2000 - Vscode.commands.executeCommand ("workbench.action.debug.continue") |> ignore - } - |> ignore - type DebugTests = | Debug | NoDebug @@ -653,7 +653,7 @@ module DotnetCli = match tryGetDebugProcessId (string consoleOutput) with | None -> () | Some processId -> - launchDebugger processId + VSCodeActions.launchDebugger processId isDebuggerStarted <- true Process.execWithCancel "dotnet" (ResizeArray(args)) (getEnv true) tryLaunchDebugger cancellationToken @@ -1627,7 +1627,11 @@ module Interactions = TestRun.showError testRun message projectRunRequest.Tests } - + module TestRunRequest = + let isDebugRequested (runRequest: TestRunRequest) = + runRequest.profile + |> Option.map (fun p -> p.kind = TestRunProfileKind.Debug) + |> Option.defaultValue false let private filtersToProjectRunRequests (rootTestCollection: TestItemCollection) @@ -1667,10 +1671,7 @@ module Interactions = [| testItem |]) |> Array.distinctBy TestItem.getId - let shouldDebug = - runRequest.profile - |> Option.map (fun p -> p.kind = TestRunProfileKind.Debug) - |> Option.defaultValue false + let shouldDebug = TestRunRequest.isDebugRequested runRequest let hasIncludeFilter = let isOnlyProjectSelected = @@ -1798,9 +1799,15 @@ module Interactions = with ex -> logger.Debug("Threw error while mapping active test items to the explorer", ex) - let incrementalUpdateHandler (runUpdate: TestRunUpdate) = - showStarted runUpdate.ActiveTests - mergeResults TrimMissing.NoTrim runUpdate.TestResults + let incrementalUpdateHandler (runUpdate: TestRunUpdateNotification) = + match runUpdate with + | Progress progress -> + logger.Debug("Nya: matched progress update", progress) + showStarted progress.ActiveTests + mergeResults TrimMissing.NoTrim progress.TestResults + | ProcessWaitingForDebugger processId -> + logger.Debug("Nya: attach debugger", processId) + VSCodeActions.launchDebugger processId let filterExpression = match req.``include`` with @@ -1814,7 +1821,11 @@ module Interactions = logger.Debug($"Test Filter Expression: {filterExpression}") - let! runResult = LanguageService.runTests incrementalUpdateHandler filterExpression + let attachDebugger = TestRunRequest.isDebugRequested req + logger.Debug("Nya: should debug", attachDebugger) + + let! runResult = + LanguageService.runTests incrementalUpdateHandler filterExpression attachDebugger mergeResults TrimMissing.Trim runResult.Data diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index f716a0f3..49225cc2 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -402,10 +402,13 @@ module DTO = type TestDiscoveryUpdate = { Tests: TestItemDTO array } - type TestRunUpdate = + type TestRunProgress = { TestResults: TestResultDTO array ActiveTests: TestItemDTO array } + type TestRunUpdateNotification = + | Progress of TestRunProgress + | ProcessWaitingForDebugger of processId: string type Result<'T> = { Kind: string; Data: 'T } diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index c2a935e4..8c498ff1 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -94,7 +94,9 @@ module LanguageService = { start: Fable.Import.VSCode.Vscode.Position ``end``: Fable.Import.VSCode.Vscode.Position } - type TestRunRequest = { TestCaseFilter: string option } + type TestRunRequest = + { TestCaseFilter: string option + AttachDebugger: bool } type Uri with @@ -611,18 +613,37 @@ Consider: cl.sendRequest ("test/discoverTests", ()) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) - let runTests incrementalUpdateHandler (testCaseFilter: string option) = + let runTests + (incrementalUpdateHandler: TestRunUpdateNotification -> unit) + (testCaseFilter: string option) + (attachDebugger: bool) + = match client with | None -> Promise.empty | Some cl -> cl.onNotification ( - "test/testRunUpdate", + "test/testRunProgressUpdate", (fun (notification: Types.PlainNotification) -> - let parsed = ofJson notification.content - incrementalUpdateHandler parsed) + logger.Debug("Nya: raw update Notification", notification) + let parsed = ofJson notification.content + logger.Debug("Nya: parsed update Notification", parsed) + incrementalUpdateHandler (Progress parsed)) + ) + + cl.onNotification ( + "test/processWaitingForDebugger", + (fun (notification: Types.PlainNotification) -> + logger.Debug("Nya: raw update Notification", notification) + let parsed = ofJson notification.content + logger.Debug("Nya: parsed update Notification", parsed) + incrementalUpdateHandler (ProcessWaitingForDebugger parsed)) ) - let request: Types.TestRunRequest = { TestCaseFilter = testCaseFilter } + let request: Types.TestRunRequest = + { TestCaseFilter = testCaseFilter + AttachDebugger = attachDebugger } + + logger.Debug("Nya: runTests request", request) cl.sendRequest ("test/runTests", request) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) From 9d807eb74632926667c176dab575771f9c8488ac Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Thu, 7 Aug 2025 13:57:01 -0500 Subject: [PATCH 13/32] Use vstest environment variable for skipping entry breakpoints instead of waiting and simulating a user continue command --- src/Components/TestExplorer.fs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index d84b8c87..c685b732 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -520,17 +520,8 @@ module VSCodeActions = let folder = workspace.workspaceFolders.Value.[0] - promise { - let! _ = - Vscode.debug.startDebugging (Some folder, U2.Case2 launchRequest) - |> Promise.ofThenable - - // NOTE: Have to wait or it'll continue before the debugger reaches the stop on entry point. - // That'll leave the debugger in a confusing state where it shows it's attached but - // no breakpoints are hit and the breakpoints show as disabled - do! Promise.sleep 2000 - Vscode.commands.executeCommand ("workbench.action.debug.continue") |> ignore - } + Vscode.debug.startDebugging (Some folder, U2.Case2 launchRequest) + |> Promise.ofThenable |> ignore @@ -639,7 +630,12 @@ module DotnetCli = let childEnv = parentEnv //NOTE: Important to include VSTEST_HOST_DEBUG=0 when not debugging to remove stale values // that may cause the debugger to wait and hang - childEnv?VSTEST_HOST_DEBUG <- (if enableTestHostDebugger then 1 else 0) + if enableTestHostDebugger then + childEnv?VSTEST_HOST_DEBUG <- 1 + childEnv?VSTEST_DEBUG_NOBP <- 1 + else + childEnv?VSTEST_HOST_DEBUG <- 0 + childEnv?VSTEST_DEBUG_NOBP <- 0 childEnv |> box |> Some match shouldDebug with From 1a1aeb2d19b2333b80d0e17dc12f4a5f7a4369fc Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 2 Sep 2025 11:12:41 -0500 Subject: [PATCH 14/32] Debug test runs using the language server --- src/Components/TestExplorer.fs | 20 +++++++++++--------- src/Core/DTO.fs | 4 +--- src/Core/LanguageService.fs | 18 +++++++++--------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index c685b732..8df28f42 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -509,7 +509,7 @@ module TrxParser = module VSCodeActions = - let launchDebugger processId = + let launchDebugger (processId: string) = let launchRequest: DebugConfiguration = {| name = ".NET Core Attach" ``type`` = "coreclr" @@ -522,7 +522,6 @@ module VSCodeActions = Vscode.debug.startDebugging (Some folder, U2.Case2 launchRequest) |> Promise.ofThenable - |> ignore module DotnetCli = @@ -636,6 +635,7 @@ module DotnetCli = else childEnv?VSTEST_HOST_DEBUG <- 0 childEnv?VSTEST_DEBUG_NOBP <- 0 + childEnv |> box |> Some match shouldDebug with @@ -1798,12 +1798,11 @@ module Interactions = let incrementalUpdateHandler (runUpdate: TestRunUpdateNotification) = match runUpdate with | Progress progress -> - logger.Debug("Nya: matched progress update", progress) showStarted progress.ActiveTests mergeResults TrimMissing.NoTrim progress.TestResults - | ProcessWaitingForDebugger processId -> - logger.Debug("Nya: attach debugger", processId) - VSCodeActions.launchDebugger processId + + let onAttachDebugger (processId: int) = + VSCodeActions.launchDebugger (string processId) let filterExpression = match req.``include`` with @@ -1817,11 +1816,14 @@ module Interactions = logger.Debug($"Test Filter Expression: {filterExpression}") - let attachDebugger = TestRunRequest.isDebugRequested req - logger.Debug("Nya: should debug", attachDebugger) + let shouldDebug = TestRunRequest.isDebugRequested req let! runResult = - LanguageService.runTests incrementalUpdateHandler filterExpression attachDebugger + LanguageService.runTests + incrementalUpdateHandler + onAttachDebugger + filterExpression + shouldDebug mergeResults TrimMissing.Trim runResult.Data diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index 49225cc2..1466e2f6 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -406,9 +406,7 @@ module DTO = { TestResults: TestResultDTO array ActiveTests: TestItemDTO array } - type TestRunUpdateNotification = - | Progress of TestRunProgress - | ProcessWaitingForDebugger of processId: string + type TestRunUpdateNotification = Progress of TestRunProgress type Result<'T> = { Kind: string; Data: 'T } diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index 8c498ff1..982d9087 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -613,8 +613,12 @@ Consider: cl.sendRequest ("test/discoverTests", ()) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) + type ProcessId = int + type DidDebuggerAttach = bool + let runTests (incrementalUpdateHandler: TestRunUpdateNotification -> unit) + (onAttachDebugger: ProcessId -> JS.Promise) (testCaseFilter: string option) (attachDebugger: bool) = @@ -624,27 +628,23 @@ Consider: cl.onNotification ( "test/testRunProgressUpdate", (fun (notification: Types.PlainNotification) -> - logger.Debug("Nya: raw update Notification", notification) let parsed = ofJson notification.content - logger.Debug("Nya: parsed update Notification", parsed) incrementalUpdateHandler (Progress parsed)) ) - cl.onNotification ( + cl.onRequest ( "test/processWaitingForDebugger", (fun (notification: Types.PlainNotification) -> - logger.Debug("Nya: raw update Notification", notification) - let parsed = ofJson notification.content - logger.Debug("Nya: parsed update Notification", parsed) - incrementalUpdateHandler (ProcessWaitingForDebugger parsed)) + promise { + let parsed = ofJson notification.content + return! onAttachDebugger parsed + }) ) let request: Types.TestRunRequest = { TestCaseFilter = testCaseFilter AttachDebugger = attachDebugger } - logger.Debug("Nya: runTests request", request) - cl.sendRequest ("test/runTests", request) |> Promise.map (fun (res: Types.PlainNotification) -> res.content |> ofJson) From d539fe57f8ad374d980eb46b27a2a871b7e551a4 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 2 Sep 2025 13:42:35 -0500 Subject: [PATCH 15/32] Forward test logs to the vscode test run output --- src/Components/TestExplorer.fs | 19 +++++++------------ src/Core/DTO.fs | 5 ++--- src/Core/LanguageService.fs | 8 ++++---- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 8df28f42..9665fb8c 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1795,11 +1795,10 @@ module Interactions = with ex -> logger.Debug("Threw error while mapping active test items to the explorer", ex) - let incrementalUpdateHandler (runUpdate: TestRunUpdateNotification) = - match runUpdate with - | Progress progress -> - showStarted progress.ActiveTests - mergeResults TrimMissing.NoTrim progress.TestResults + let onTestRunProgress (progress: TestRunProgress) = + showStarted progress.ActiveTests + mergeResults TrimMissing.NoTrim progress.TestResults + progress.TestLogs |> Array.iter (TestRun.appendOutputLine testRun) let onAttachDebugger (processId: int) = VSCodeActions.launchDebugger (string processId) @@ -1819,11 +1818,7 @@ module Interactions = let shouldDebug = TestRunRequest.isDebugRequested req let! runResult = - LanguageService.runTests - incrementalUpdateHandler - onAttachDebugger - filterExpression - shouldDebug + LanguageService.runTests onTestRunProgress onAttachDebugger filterExpression shouldDebug mergeResults TrimMissing.Trim runResult.Data @@ -2005,7 +2000,7 @@ module Interactions = let mutable discoveredTestCount: int = 0 - let incrementalUpdateHandler (discoveryUpdate: TestDiscoveryUpdate) : unit = + let onTestDiscoveryProgress (discoveryUpdate: TestDiscoveryUpdate) : unit = try let newItems = discoveryUpdate.Tests |> TestItem.ofTestDTOs testItemFactory tryGetLocation @@ -2019,7 +2014,7 @@ module Interactions = logger.Debug("Incremental test discovery update threw an exception", e) report "Discovering tests" - let! discoveryResponse = LanguageService.discoverTests incrementalUpdateHandler () + let! discoveryResponse = LanguageService.discoverTests onTestDiscoveryProgress () let testItems = discoveryResponse.Data diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index 1466e2f6..38c53fb0 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -403,11 +403,10 @@ module DTO = type TestDiscoveryUpdate = { Tests: TestItemDTO array } type TestRunProgress = - { TestResults: TestResultDTO array + { TestLogs: string array + TestResults: TestResultDTO array ActiveTests: TestItemDTO array } - type TestRunUpdateNotification = Progress of TestRunProgress - type Result<'T> = { Kind: string; Data: 'T } type HelptextResult = Result diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index 982d9087..d91564c7 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -599,7 +599,7 @@ Consider: let fsiSdk () = promise { return Environment.configFsiSdkFilePath () } - let discoverTests incrementalUpdateHandler () = + let discoverTests onDiscoveryProgress () = match client with | None -> Promise.empty | Some cl -> @@ -607,7 +607,7 @@ Consider: "test/testDiscoveryUpdate", (fun (notification: Types.PlainNotification) -> let parsed = ofJson notification.content - incrementalUpdateHandler parsed) + onDiscoveryProgress parsed) ) cl.sendRequest ("test/discoverTests", ()) @@ -617,7 +617,7 @@ Consider: type DidDebuggerAttach = bool let runTests - (incrementalUpdateHandler: TestRunUpdateNotification -> unit) + (onTestRunProgress: TestRunProgress -> unit) (onAttachDebugger: ProcessId -> JS.Promise) (testCaseFilter: string option) (attachDebugger: bool) @@ -629,7 +629,7 @@ Consider: "test/testRunProgressUpdate", (fun (notification: Types.PlainNotification) -> let parsed = ofJson notification.content - incrementalUpdateHandler (Progress parsed)) + onTestRunProgress parsed) ) cl.onRequest ( From 9c66a63236eacb730343aeaa028fe9396f152d43 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Thu, 4 Sep 2025 14:42:39 -0500 Subject: [PATCH 16/32] Allow client to limit which test projects are run This was mainly motivated by test debugging. Before this feature, the debugger would launch for every test project even if you only wanted to debug one test. With this change, it'll only launch the one project. This also partially mitigates the bug where NUnit doesn't respect filters, now those test won't show unless some tests from an nunit project are selected --- src/Components/TestExplorer.fs | 25 +++++++++++++++++-------- src/Core/LanguageService.fs | 7 +++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 9665fb8c..ddae9262 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1803,22 +1803,31 @@ module Interactions = let onAttachDebugger (processId: int) = VSCodeActions.launchDebugger (string processId) - let filterExpression = + let filterExpression, projectSubset = match req.``include`` with - | None -> None - | Some selectedCases when Seq.isEmpty selectedCases -> None + | None -> None, None + | Some selectedCases when Seq.isEmpty selectedCases -> None, None | Some selectedCases -> - projectRunRequests - |> Array.collect (fun rr -> rr.Tests) - |> buildFilterExpression - |> Some + let filter = + projectRunRequests + |> Array.collect (fun rr -> rr.Tests) + |> buildFilterExpression + |> Some + + let projectSubset = projectRunRequests |> Array.map (fun p -> p.ProjectPath) |> Some + filter, projectSubset logger.Debug($"Test Filter Expression: {filterExpression}") let shouldDebug = TestRunRequest.isDebugRequested req let! runResult = - LanguageService.runTests onTestRunProgress onAttachDebugger filterExpression shouldDebug + LanguageService.runTests + onTestRunProgress + onAttachDebugger + projectSubset + filterExpression + shouldDebug mergeResults TrimMissing.Trim runResult.Data diff --git a/src/Core/LanguageService.fs b/src/Core/LanguageService.fs index d91564c7..071e2149 100644 --- a/src/Core/LanguageService.fs +++ b/src/Core/LanguageService.fs @@ -95,7 +95,8 @@ module LanguageService = ``end``: Fable.Import.VSCode.Vscode.Position } type TestRunRequest = - { TestCaseFilter: string option + { LimitToProjects: string array option + TestCaseFilter: string option AttachDebugger: bool } type Uri with @@ -619,6 +620,7 @@ Consider: let runTests (onTestRunProgress: TestRunProgress -> unit) (onAttachDebugger: ProcessId -> JS.Promise) + (projectSubset: string array option) (testCaseFilter: string option) (attachDebugger: bool) = @@ -642,7 +644,8 @@ Consider: ) let request: Types.TestRunRequest = - { TestCaseFilter = testCaseFilter + { LimitToProjects = projectSubset + TestCaseFilter = testCaseFilter AttachDebugger = attachDebugger } cl.sendRequest ("test/runTests", request) From d6b45ea211229863fd72e73de688031b3232bdd6 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 9 Sep 2025 15:54:27 -0500 Subject: [PATCH 17/32] Show a warning if test explorer refresh doesn't find any tests --- src/Components/TestExplorer.fs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index ddae9262..aad57bba 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -2031,6 +2031,12 @@ module Interactions = |> ResizeArray rootTestCollection.replace (testItems) + + if testItems |> Seq.length = 0 then + window.showWarningMessage ( + $"No tests discovered. Make sure your projects are restored, built, and can be run with dotnet test." + ) + |> ignore } let tryMatchTestBySuffix (locationCache: CodeLocationCache) (testId: TestId) = From 27357e7b93b316b568117051bc2a0d5fbd1bdc95 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 9 Sep 2025 17:05:20 -0500 Subject: [PATCH 18/32] Forward test discovery logs to the test adapter logs for improved error diagnostics --- src/Components/TestExplorer.fs | 14 ++++++++++++-- src/Core/DTO.fs | 16 ++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index aad57bba..883a9c4a 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1798,7 +1798,8 @@ module Interactions = let onTestRunProgress (progress: TestRunProgress) = showStarted progress.ActiveTests mergeResults TrimMissing.NoTrim progress.TestResults - progress.TestLogs |> Array.iter (TestRun.appendOutputLine testRun) + let formatLog (log: TestLogMessage) = $"[{log.Level}] {log.Message}" + progress.TestLogs |> Array.iter (formatLog >> TestRun.appendOutputLine testRun) let onAttachDebugger (processId: int) = VSCodeActions.launchDebugger (string processId) @@ -2010,7 +2011,16 @@ module Interactions = let mutable discoveredTestCount: int = 0 let onTestDiscoveryProgress (discoveryUpdate: TestDiscoveryUpdate) : unit = + let writeTestLog (log: TestLogMessage) = + let message = $"[Discover Tests] {log.Message}" + match log.Level with + | TestLogLevel.Warning -> logger.Warn(message) + | TestLogLevel.Error -> logger.Error(message) + | TestLogLevel.Informational -> logger.Info(message) + try + discoveryUpdate.TestLogs |> Array.iter writeTestLog + let newItems = discoveryUpdate.Tests |> TestItem.ofTestDTOs testItemFactory tryGetLocation @@ -2034,7 +2044,7 @@ module Interactions = if testItems |> Seq.length = 0 then window.showWarningMessage ( - $"No tests discovered. Make sure your projects are restored, built, and can be run with dotnet test." + $"No tests discovered. Make sure your projects are restored, built, and can be run with dotnet test. Discovery logs can be found in Output > F# - Test Adapter " ) |> ignore } diff --git a/src/Core/DTO.fs b/src/Core/DTO.fs index 38c53fb0..ddac7126 100644 --- a/src/Core/DTO.fs +++ b/src/Core/DTO.fs @@ -400,10 +400,22 @@ module DTO = AdditionalOutput: string option Duration: System.TimeSpan } - type TestDiscoveryUpdate = { Tests: TestItemDTO array } + [] + [] + type TestLogLevel = + | Informational + | Warning + | Error + + type TestLogMessage = + { Level: TestLogLevel; Message: string } + + type TestDiscoveryUpdate = + { Tests: TestItemDTO array + TestLogs: TestLogMessage array } type TestRunProgress = - { TestLogs: string array + { TestLogs: TestLogMessage array TestResults: TestResultDTO array ActiveTests: TestItemDTO array } From 28d39f7c79da0ef2147705e3752516096361bf3e Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 10 Sep 2025 17:45:41 -0500 Subject: [PATCH 19/32] Colorize log levels in the TestRun output --- src/Components/TestExplorer.fs | 38 ++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 883a9c4a..8e25413c 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1335,9 +1335,21 @@ module Interactions = let normalizeLineEndings str = RegularExpressions.Regex.Replace(str, @"\r\n|\n\r|\n|\r", "\r\n") - let appendOutputLine (testRun: TestRun) (message: string) = - // NOTE: New lines must be crlf https://code.visualstudio.com/api/extension-guides/testing#test-output - testRun.appendOutput (sprintf "%s\r\n" (normalizeLineEndings message)) + module Output = + module private Ansi = + let yellow (text: string) = $"\u001B[33m{text}\u001B[0m" + let green (text: string) = $"\u001B[32m{text}\u001B[0m" + let red (text: string) = $"\u001B[31m{text}\u001B[0m" + + let appendLine (testRun: TestRun) (message: string) = + // NOTE: New lines must be crlf https://code.visualstudio.com/api/extension-guides/testing#test-output + testRun.appendOutput (sprintf "%s\r\n" (normalizeLineEndings message)) + + let appendWarningLine (testRun: TestRun) (message: string) = + appendLine testRun (message |> Ansi.yellow) + + let appendErrorLine (testRun: TestRun) (message: string) = + appendLine testRun (message |> Ansi.red) let appendOutputLineForTest (testRun: TestRun) (testItem) (message: string) = let message = sprintf "%s\r\n" (normalizeLineEndings message) @@ -1578,7 +1590,7 @@ module Interactions = (projectRunRequest.ShouldDebug |> DotnetCli.DebugTests.ofBool) cancellationToken - TestRun.appendOutputLine testRun output + TestRun.Output.appendLine testRun output let testResults = TrxParser.extractTrxResults trxPath |> Array.map trxResultToTestResult @@ -1588,7 +1600,7 @@ module Interactions = $"WARNING: No tests ran for project \"{projectPath}\". \r\nThe test explorer might be out of sync. Try running a higher test or refreshing the test explorer" window.showWarningMessage (message) |> ignore - TestRun.appendOutputLine testRun message + TestRun.Output.appendWarningLine testRun message else mergeResultsToExplorer testRun @@ -1619,7 +1631,7 @@ module Interactions = let message = $"❌ Error running tests: \n project: {projectRunRequest.ProjectPath} \n\n error:\n {e.Message}" - TestRun.appendOutputLine testRun message + TestRun.Output.appendErrorLine testRun message TestRun.showError testRun message projectRunRequest.Tests } @@ -1721,7 +1733,7 @@ module Interactions = if buildStatus.Code <> Some 0 then TestRun.showError testRun "Project build failed" runnableTests - TestRun.appendOutputLine testRun $"❌ Failed to build project: {projectPath}" + TestRun.Output.appendErrorLine testRun $"❌ Failed to build project: {projectPath}" return None else return Some projectRunRequest @@ -1798,8 +1810,15 @@ module Interactions = let onTestRunProgress (progress: TestRunProgress) = showStarted progress.ActiveTests mergeResults TrimMissing.NoTrim progress.TestResults - let formatLog (log: TestLogMessage) = $"[{log.Level}] {log.Message}" - progress.TestLogs |> Array.iter (formatLog >> TestRun.appendOutputLine testRun) + + let appendToTestRun testRun (log: TestLogMessage) = + match log.Level with + | TestLogLevel.Informational -> TestRun.Output.appendLine testRun log.Message + | TestLogLevel.Warning -> + TestRun.Output.appendWarningLine testRun $"[WARN] {log.Message}" + | TestLogLevel.Error -> TestRun.Output.appendErrorLine testRun $"[ERROR] {log.Message}" + + progress.TestLogs |> Array.iter (appendToTestRun testRun) let onAttachDebugger (processId: int) = VSCodeActions.launchDebugger (string processId) @@ -2013,6 +2032,7 @@ module Interactions = let onTestDiscoveryProgress (discoveryUpdate: TestDiscoveryUpdate) : unit = let writeTestLog (log: TestLogMessage) = let message = $"[Discover Tests] {log.Message}" + match log.Level with | TestLogLevel.Warning -> logger.Warn(message) | TestLogLevel.Error -> logger.Error(message) From 1cfab84e12ed6997fca40aa508e2d62ca08add2f Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:00:45 -0500 Subject: [PATCH 20/32] Log test outcomes to the TestRun output --- src/Components/TestExplorer.fs | 36 ++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 8e25413c..23bc410f 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1341,6 +1341,11 @@ module Interactions = let green (text: string) = $"\u001B[32m{text}\u001B[0m" let red (text: string) = $"\u001B[31m{text}\u001B[0m" + module Symbols = + let testPassed = Ansi.green "Passed" + let testFailed = Ansi.red "Failed" + let testSkipped = Ansi.yellow "Skipped" + let appendLine (testRun: TestRun) (message: string) = // NOTE: New lines must be crlf https://code.visualstudio.com/api/extension-guides/testing#test-output testRun.appendOutput (sprintf "%s\r\n" (normalizeLineEndings message)) @@ -1811,12 +1816,39 @@ module Interactions = showStarted progress.ActiveTests mergeResults TrimMissing.NoTrim progress.TestResults + let appendTestResultToOutput (testResult: TestResultDTO) = + match testResult.Outcome with + | TestOutcomeDTO.Passed -> + TestRun.Output.appendLine + testRun + $"{TestRun.Output.Symbols.testPassed} {testResult.TestItem.FullName}" + | TestOutcomeDTO.Failed -> + TestRun.Output.appendLine + testRun + $"{TestRun.Output.Symbols.testFailed} {testResult.TestItem.FullName}" + | TestOutcomeDTO.Skipped -> + TestRun.Output.appendLine + testRun + $"{TestRun.Output.Symbols.testSkipped} {testResult.TestItem.FullName}" + | TestOutcomeDTO.None -> + TestRun.Output.appendWarningLine + testRun + $"No outcome for {testResult.TestItem.FullName}" + | TestOutcomeDTO.NotFound -> + TestRun.Output.appendWarningLine testRun $"NotFound {testResult.TestItem.FullName}" + | _ -> + TestRun.Output.appendWarningLine + testRun + $"An unexpected test outcome was encountered for {testResult.TestItem.FullName}" + + progress.TestResults |> Array.iter appendTestResultToOutput + let appendToTestRun testRun (log: TestLogMessage) = match log.Level with | TestLogLevel.Informational -> TestRun.Output.appendLine testRun log.Message | TestLogLevel.Warning -> - TestRun.Output.appendWarningLine testRun $"[WARN] {log.Message}" - | TestLogLevel.Error -> TestRun.Output.appendErrorLine testRun $"[ERROR] {log.Message}" + TestRun.Output.appendWarningLine testRun log.Message + | TestLogLevel.Error -> TestRun.Output.appendErrorLine testRun log.Message progress.TestLogs |> Array.iter (appendToTestRun testRun) From e1d716f1b4e1e622f98fe3332e8b24120957fd34 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:24:46 -0500 Subject: [PATCH 21/32] Raise a warning if no test ran for a test run --- src/Components/TestExplorer.fs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 23bc410f..f031a753 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1846,8 +1846,7 @@ module Interactions = let appendToTestRun testRun (log: TestLogMessage) = match log.Level with | TestLogLevel.Informational -> TestRun.Output.appendLine testRun log.Message - | TestLogLevel.Warning -> - TestRun.Output.appendWarningLine testRun log.Message + | TestLogLevel.Warning -> TestRun.Output.appendWarningLine testRun log.Message | TestLogLevel.Error -> TestRun.Output.appendErrorLine testRun log.Message progress.TestLogs |> Array.iter (appendToTestRun testRun) @@ -1883,6 +1882,13 @@ module Interactions = mergeResults TrimMissing.Trim runResult.Data + if Array.isEmpty runResult.Data then + let message = + $"WARNING: No tests ran. The test explorer might be out of sync. Try running a higher test group or refreshing the test explorer" + + window.showWarningMessage (message) |> ignore + TestRun.Output.appendWarningLine testRun message + with ex -> logger.Debug("Test run failed with exception", ex) From 9ec3d112f69ba941686416600f03e9d9d4616aa8 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Thu, 11 Sep 2025 13:13:09 -0500 Subject: [PATCH 22/32] Factor the language server discovery and run flows to clarify the top-level handlers Also enables experimentation with running discovery before a test run to make the explorer more robust for projects without live update support --- src/Components/TestExplorer.fs | 407 +++++++++++++++++---------------- 1 file changed, 215 insertions(+), 192 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index f031a753..c37fd8f7 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -649,7 +649,7 @@ module DotnetCli = match tryGetDebugProcessId (string consoleOutput) with | None -> () | Some processId -> - VSCodeActions.launchDebugger processId + VSCodeActions.launchDebugger processId |> ignore isDebuggerStarted <- true Process.execWithCancel "dotnet" (ResizeArray(args)) (getEnv true) tryLaunchDebugger cancellationToken @@ -1159,7 +1159,6 @@ module TestDiscovery = (replacementItem, withUri) let rec recurse (target: TestItemCollection) (withUri: TestItem array) : unit = - let treeOnly, matched, _codeOnly = ArrayExt.venn TestItem.getId TestItem.getId (target.TestItems()) withUri @@ -1667,8 +1666,7 @@ module Interactions = let message = $"Could not run tests: project not loaded. {projectPath}" invalidOp message) |> Option.defaultWith (fun () -> - let message = - $"Could not run tests: project does not found in workspace. {projectPath}" + let message = $"Could not run tests: project not found in workspace. {projectPath}" logger.Error(message) invalidOp message) @@ -1700,6 +1698,215 @@ module Interactions = HasIncludeFilter = hasIncludeFilter Tests = replaceProjectRootIfPresent tests }) + let private discoverTests_WithLangaugeServer + testItemFactory + (rootTestCollection: TestItemCollection) + tryGetLocation + = + withProgress NoCancel + <| fun p progressCancelToken -> + promise { + let report message = + logger.Info message + + p.report + {| message = Some message + increment = None |} + + let mergeTestItemCollections (target: TestItem array) (addition: TestItem array) : TestItem array = + let rec recurse (target: TestItem array) (addition: TestItem array) : TestItem array = + let targetOnly, conficted, addedOnly = + ArrayExt.venn TestItem.getId TestItem.getId target addition + + let mergeSingle (targetItem: TestItem, addedItem: TestItem) = + let mergedChildren = + recurse (targetItem.children.TestItems()) (addedItem.children.TestItems()) + + addedItem.children.replace (ResizeArray mergedChildren) + addedItem + + Array.concat [ targetOnly; addedOnly; conficted |> Array.map mergeSingle ] + + recurse target addition + + let mutable discoveredTestsAccumulator: TestItem array = + rootTestCollection.TestItems() + + let mutable discoveredTestCount: int = 0 + + let onTestDiscoveryProgress (discoveryUpdate: TestDiscoveryUpdate) : unit = + let writeTestLog (log: TestLogMessage) = + let message = $"[Discover Tests] {log.Message}" + + match log.Level with + | TestLogLevel.Warning -> logger.Warn(message) + | TestLogLevel.Error -> logger.Error(message) + | TestLogLevel.Informational -> logger.Info(message) + + try + discoveryUpdate.TestLogs |> Array.iter writeTestLog + + let newItems = + discoveryUpdate.Tests |> TestItem.ofTestDTOs testItemFactory tryGetLocation + + discoveredTestsAccumulator <- mergeTestItemCollections discoveredTestsAccumulator newItems + discoveredTestCount <- discoveredTestCount + (discoveryUpdate.Tests |> Array.length) + + report $"Discovering tests: {discoveredTestCount} discovered" + rootTestCollection.replace (ResizeArray discoveredTestsAccumulator) + with e -> + logger.Debug("Incremental test discovery update threw an exception", e) + + report "Discovering tests" + let! discoveryResponse = LanguageService.discoverTests onTestDiscoveryProgress () + + let testItems = + discoveryResponse.Data + |> TestItem.ofTestDTOs testItemFactory tryGetLocation + |> ResizeArray + + rootTestCollection.replace (testItems) + + if testItems |> Seq.length = 0 then + window.showWarningMessage ( + $"No tests discovered. Make sure your projects are restored, built, and can be run with dotnet test. Discovery logs can be found in Output > F# - Test Adapter " + ) + |> ignore + } + + let private runTests_WithLanguageServer + mergeTestResultsToExplorer + (req: TestRunRequest) + testRun + projectRunRequests + = + promise { + try + let runnableTestsByProject = + projectRunRequests + |> Array.map (fun rr -> rr.ProjectPath, rr.Tests |> Array.collect TestItem.runnableChildren) + |> Map + + let expectedTestsById = + runnableTestsByProject + |> Map.values + |> Seq.collect (Seq.map (fun t -> t.id, t)) + |> Map + + let mergeResults (shouldTrim: TrimMissing) (resultDtos: TestResultDTO array) = + let groups = + resultDtos + |> Array.groupBy (fun tr -> tr.TestItem.ProjectFilePath, tr.TestItem.TargetFramework) + + groups + |> Array.iter (fun ((projPath, targetFramework), results) -> + let expectedToRun = + runnableTestsByProject + |> Map.tryFind projPath + |> Option.defaultValue Array.empty + + let actuallyRan: TestResult array = results |> Array.map TestResult.ofTestResultDTO + + mergeTestResultsToExplorer + testRun + projPath + targetFramework + shouldTrim + expectedToRun + actuallyRan) + + let showStarted (testItems: TestItemDTO array) = + try + let groups = testItems |> Array.groupBy (fun t -> t.ProjectFilePath) + + groups + |> Array.iter (fun (projPath, activeTests) -> + let testIdsToStart = + activeTests |> Array.map (fun t -> TestItem.constructId projPath t.FullName) + + let knownExplorerItems = testIdsToStart |> Array.choose expectedTestsById.TryFind + knownExplorerItems |> TestRun.showStarted testRun) + with ex -> + logger.Debug("Threw error while mapping active test items to the explorer", ex) + + let onTestRunProgress (progress: TestRunProgress) = + showStarted progress.ActiveTests + mergeResults TrimMissing.NoTrim progress.TestResults + + let appendTestResultToOutput (testResult: TestResultDTO) = + match testResult.Outcome with + | TestOutcomeDTO.Passed -> + TestRun.Output.appendLine + testRun + $"{TestRun.Output.Symbols.testPassed} {testResult.TestItem.FullName}" + | TestOutcomeDTO.Failed -> + TestRun.Output.appendLine + testRun + $"{TestRun.Output.Symbols.testFailed} {testResult.TestItem.FullName}" + | TestOutcomeDTO.Skipped -> + TestRun.Output.appendLine + testRun + $"{TestRun.Output.Symbols.testSkipped} {testResult.TestItem.FullName}" + | TestOutcomeDTO.None -> + TestRun.Output.appendWarningLine testRun $"No outcome for {testResult.TestItem.FullName}" + | TestOutcomeDTO.NotFound -> + TestRun.Output.appendWarningLine testRun $"NotFound {testResult.TestItem.FullName}" + | _ -> + TestRun.Output.appendWarningLine + testRun + $"An unexpected test outcome was encountered for {testResult.TestItem.FullName}" + + progress.TestResults |> Array.iter appendTestResultToOutput + + let appendToTestRun testRun (log: TestLogMessage) = + match log.Level with + | TestLogLevel.Informational -> TestRun.Output.appendLine testRun log.Message + | TestLogLevel.Warning -> TestRun.Output.appendWarningLine testRun log.Message + | TestLogLevel.Error -> TestRun.Output.appendErrorLine testRun log.Message + + progress.TestLogs |> Array.iter (appendToTestRun testRun) + + let onAttachDebugger (processId: int) = + VSCodeActions.launchDebugger (string processId) + + let filterExpression, projectSubset = + match req.``include`` with + | None -> None, None + | Some selectedCases when Seq.isEmpty selectedCases -> None, None + | Some selectedCases -> + let filter = + projectRunRequests + |> Array.collect (fun rr -> rr.Tests) + |> buildFilterExpression + |> Some + + let projectSubset = projectRunRequests |> Array.map (fun p -> p.ProjectPath) |> Some + filter, projectSubset + + logger.Debug($"Test Filter Expression: {filterExpression}") + + let shouldDebug = TestRunRequest.isDebugRequested req + + let! runResult = + LanguageService.runTests + onTestRunProgress + onAttachDebugger + projectSubset + filterExpression + shouldDebug + + mergeResults TrimMissing.Trim runResult.Data + + if Array.isEmpty runResult.Data then + let message = + $"WARNING: No tests ran. The test explorer might be out of sync. Try running a higher test group or refreshing the test explorer" + + window.showWarningMessage (message) |> ignore + TestRun.Output.appendWarningLine testRun message + with ex -> + logger.Debug("Test run failed with exception", ex) + } + let runHandler (testController: TestController) (tryGetLocation: TestId -> LocationRecord option) @@ -1764,139 +1971,13 @@ module Interactions = () else - try - let runnableTestsByProject = - projectRunRequests - |> Array.map (fun rr -> rr.ProjectPath, rr.Tests |> Array.collect TestItem.runnableChildren) - |> Map - - let expectedTestsById = - runnableTestsByProject - |> Map.values - |> Seq.collect (Seq.map (fun t -> t.id, t)) - |> Map - - let mergeResults (shouldTrim: TrimMissing) (resultDtos: TestResultDTO array) = - let groups = - resultDtos - |> Array.groupBy (fun tr -> tr.TestItem.ProjectFilePath, tr.TestItem.TargetFramework) - - groups - |> Array.iter (fun ((projPath, targetFramework), results) -> - let expectedToRun = - runnableTestsByProject - |> Map.tryFind projPath - |> Option.defaultValue Array.empty - - let actuallyRan: TestResult array = results |> Array.map TestResult.ofTestResultDTO - - mergeTestResultsToExplorer - testRun - projPath - targetFramework - shouldTrim - expectedToRun - actuallyRan) - - let showStarted (testItems: TestItemDTO array) = - try - let groups = testItems |> Array.groupBy (fun t -> t.ProjectFilePath) - - groups - |> Array.iter (fun (projPath, activeTests) -> - let testIdsToStart = - activeTests |> Array.map (fun t -> TestItem.constructId projPath t.FullName) - - let knownExplorerItems = testIdsToStart |> Array.choose expectedTestsById.TryFind - knownExplorerItems |> TestRun.showStarted testRun) - with ex -> - logger.Debug("Threw error while mapping active test items to the explorer", ex) - - let onTestRunProgress (progress: TestRunProgress) = - showStarted progress.ActiveTests - mergeResults TrimMissing.NoTrim progress.TestResults - - let appendTestResultToOutput (testResult: TestResultDTO) = - match testResult.Outcome with - | TestOutcomeDTO.Passed -> - TestRun.Output.appendLine - testRun - $"{TestRun.Output.Symbols.testPassed} {testResult.TestItem.FullName}" - | TestOutcomeDTO.Failed -> - TestRun.Output.appendLine - testRun - $"{TestRun.Output.Symbols.testFailed} {testResult.TestItem.FullName}" - | TestOutcomeDTO.Skipped -> - TestRun.Output.appendLine - testRun - $"{TestRun.Output.Symbols.testSkipped} {testResult.TestItem.FullName}" - | TestOutcomeDTO.None -> - TestRun.Output.appendWarningLine - testRun - $"No outcome for {testResult.TestItem.FullName}" - | TestOutcomeDTO.NotFound -> - TestRun.Output.appendWarningLine testRun $"NotFound {testResult.TestItem.FullName}" - | _ -> - TestRun.Output.appendWarningLine - testRun - $"An unexpected test outcome was encountered for {testResult.TestItem.FullName}" - - progress.TestResults |> Array.iter appendTestResultToOutput - - let appendToTestRun testRun (log: TestLogMessage) = - match log.Level with - | TestLogLevel.Informational -> TestRun.Output.appendLine testRun log.Message - | TestLogLevel.Warning -> TestRun.Output.appendWarningLine testRun log.Message - | TestLogLevel.Error -> TestRun.Output.appendErrorLine testRun log.Message - - progress.TestLogs |> Array.iter (appendToTestRun testRun) - - let onAttachDebugger (processId: int) = - VSCodeActions.launchDebugger (string processId) - - let filterExpression, projectSubset = - match req.``include`` with - | None -> None, None - | Some selectedCases when Seq.isEmpty selectedCases -> None, None - | Some selectedCases -> - let filter = - projectRunRequests - |> Array.collect (fun rr -> rr.Tests) - |> buildFilterExpression - |> Some - - let projectSubset = projectRunRequests |> Array.map (fun p -> p.ProjectPath) |> Some - filter, projectSubset - - logger.Debug($"Test Filter Expression: {filterExpression}") - - let shouldDebug = TestRunRequest.isDebugRequested req - - let! runResult = - LanguageService.runTests - onTestRunProgress - onAttachDebugger - projectSubset - filterExpression - shouldDebug - - mergeResults TrimMissing.Trim runResult.Data - - if Array.isEmpty runResult.Data then - let message = - $"WARNING: No tests ran. The test explorer might be out of sync. Try running a higher test group or refreshing the test explorer" - - window.showWarningMessage (message) |> ignore - TestRun.Output.appendWarningLine testRun message - - with ex -> - logger.Debug("Test run failed with exception", ex) + do! runTests_WithLanguageServer mergeTestResultsToExplorer req testRun projectRunRequests testRun.``end`` () } |> (Promise.toThenable >> (!^)) - let private discoverTestsWithDotnetCli + let private discoverTests_WithDotnetCli testItemFactory tryGetLocation makeTrxPath @@ -2037,7 +2118,7 @@ module Interactions = else if useLegacyDotnetCliIntegration then do! - discoverTestsWithDotnetCli + discoverTests_WithDotnetCli testItemFactory tryGetLocation makeTrxPath @@ -2046,65 +2127,7 @@ module Interactions = cancellationToken builtTestProjects else - let mergeTestItemCollections (target: TestItem array) (addition: TestItem array) : TestItem array = - let rec recurse (target: TestItem array) (addition: TestItem array) : TestItem array = - let targetOnly, conficted, addedOnly = - ArrayExt.venn TestItem.getId TestItem.getId target addition - - let mergeSingle (targetItem: TestItem, addedItem: TestItem) = - let mergedChildren = - recurse (targetItem.children.TestItems()) (addedItem.children.TestItems()) - - addedItem.children.replace (ResizeArray mergedChildren) - addedItem - - Array.concat [ targetOnly; addedOnly; conficted |> Array.map mergeSingle ] - - recurse target addition - - let mutable discoveredTestsAccumulator: TestItem array = - rootTestCollection.TestItems() - - let mutable discoveredTestCount: int = 0 - - let onTestDiscoveryProgress (discoveryUpdate: TestDiscoveryUpdate) : unit = - let writeTestLog (log: TestLogMessage) = - let message = $"[Discover Tests] {log.Message}" - - match log.Level with - | TestLogLevel.Warning -> logger.Warn(message) - | TestLogLevel.Error -> logger.Error(message) - | TestLogLevel.Informational -> logger.Info(message) - - try - discoveryUpdate.TestLogs |> Array.iter writeTestLog - - let newItems = - discoveryUpdate.Tests |> TestItem.ofTestDTOs testItemFactory tryGetLocation - - discoveredTestsAccumulator <- mergeTestItemCollections discoveredTestsAccumulator newItems - discoveredTestCount <- discoveredTestCount + (discoveryUpdate.Tests |> Array.length) - - report $"Discovering tests: {discoveredTestCount} discovered" - rootTestCollection.replace (ResizeArray discoveredTestsAccumulator) - with e -> - logger.Debug("Incremental test discovery update threw an exception", e) - - report "Discovering tests" - let! discoveryResponse = LanguageService.discoverTests onTestDiscoveryProgress () - - let testItems = - discoveryResponse.Data - |> TestItem.ofTestDTOs testItemFactory tryGetLocation - |> ResizeArray - - rootTestCollection.replace (testItems) - - if testItems |> Seq.length = 0 then - window.showWarningMessage ( - $"No tests discovered. Make sure your projects are restored, built, and can be run with dotnet test. Discovery logs can be found in Output > F# - Test Adapter " - ) - |> ignore + do! discoverTests_WithLangaugeServer testItemFactory rootTestCollection tryGetLocation } let tryMatchTestBySuffix (locationCache: CodeLocationCache) (testId: TestId) = From 7547296fb358a337fc8af1336a7c424fc3e1e130 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Thu, 11 Sep 2025 14:32:02 -0500 Subject: [PATCH 23/32] Refactor mergeTestResultsToExplorer for smoother use with LSP test results The dotnet cli approach fundamentally expected tests to be run and merged by project, but VSTest doesn't. MergeTestResultsToExplorer really only needed the project and target framework for creating new items. By adding project and target framework to the each test result, we can merge test items without expecting them to be batched by project --- src/Components/TestExplorer.fs | 107 +++++++++++++++------------------ 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index c37fd8f7..8616aa2e 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -304,7 +304,9 @@ type TestResult = Expected: string option Actual: string option Timing: float - TestFramework: TestFrameworkId option } + TestFramework: TestFrameworkId option + ProjectFilePath: ProjectFilePath + TargetFramework: TargetFramework } module TestResult = let tryExtractExpectedAndActual (message: string option) = @@ -324,7 +326,7 @@ module TestResult = expected, actual - let ofTestResultDTO (testResultDto: TestResultDTO) = + let ofTestResultDTO (testResultDto: TestResultDTO) : TestResult = let expected, actual = tryExtractExpectedAndActual testResultDto.ErrorMessage { FullTestName = testResultDto.TestItem |> TestItemDTO.getNormalizedFullName @@ -335,7 +337,9 @@ module TestResult = Timing = testResultDto.Duration.Milliseconds TestFramework = testResultDto.TestItem.ExecutorUri |> TestFrameworkId.tryFromExecutorUri Expected = expected - Actual = actual } + Actual = actual + ProjectFilePath = testResultDto.TestItem.ProjectFilePath + TargetFramework = testResultDto.TestItem.TargetFramework } module Path = @@ -1487,8 +1491,6 @@ module Interactions = (testItemFactory: TestItem.TestItemFactory) (tryGetLocation: TestId -> LocationRecord option) (testRun: TestRun) - (projectPath: ProjectPath) - (targetFramework: TargetFramework) (shouldDeleteMissing: bool) (expectedToRun: TestItem array) (testResults: TestResult array) @@ -1503,18 +1505,19 @@ module Interactions = parentCollection.delete testWithoutResult.id - let getOrMakeHierarchyPath testFramework = + let getOrMakeHierarchyPath (testResult: TestResult) = let testItemFactory (ti: TestItem.TestItemBuilder) = testItemFactory { ti with - testFramework = testFramework } + testFramework = testResult.TestFramework } TestItem.getOrMakeHierarchyPath rootTestCollection testItemFactory tryGetLocation - projectPath - targetFramework + testResult.ProjectFilePath + testResult.TargetFramework + testResult.FullTestName let treeItemComparable (t: TestItem) = TestItem.getFullName t.id let resultComparable (r: TestResult) = r.FullTestName @@ -1529,12 +1532,15 @@ module Interactions = added |> Array.iter (fun additionalResult -> - let treeItem = - getOrMakeHierarchyPath additionalResult.TestFramework additionalResult.FullTestName + let treeItem = getOrMakeHierarchyPath additionalResult displayTestResultInExplorer testRun (treeItem, additionalResult)) - let private trxResultToTestResult (trxResult: TrxParser.TestWithResult) = + let private trxResultToTestResult + (projectFilePath: ProjectFilePath) + (targetFramework: TargetFramework) + (trxResult: TrxParser.TestWithResult) + = let expected, actual = TestResult.tryExtractExpectedAndActual trxResult.UnitTestResult.Output.ErrorInfo.Message @@ -1547,7 +1553,9 @@ module Interactions = Expected = expected Actual = actual Timing = trxResult.UnitTestResult.Duration.Milliseconds - TestFramework = TestFrameworkId.tryFromExecutorUri trxResult.UnitTest.TestMethod.AdapterTypeName } + TestFramework = TestFrameworkId.tryFromExecutorUri trxResult.UnitTest.TestMethod.AdapterTypeName + ProjectFilePath = projectFilePath + TargetFramework = targetFramework } type TrimMissing = bool @@ -1555,8 +1563,7 @@ module Interactions = let Trim = true let NoTrim = false - type MergeTestResultsToExplorer = - TestRun -> ProjectPath -> TargetFramework -> TrimMissing -> TestItem array -> TestResult array -> unit + type MergeTestResultsToExplorer = TestRun -> TrimMissing -> TestItem array -> TestResult array -> unit let private runTestProject_withoutExceptionHandling (mergeResultsToExplorer: MergeTestResultsToExplorer) @@ -1597,7 +1604,8 @@ module Interactions = TestRun.Output.appendLine testRun output let testResults = - TrxParser.extractTrxResults trxPath |> Array.map trxResultToTestResult + TrxParser.extractTrxResults trxPath + |> Array.map (trxResultToTestResult projectPath projectRunRequest.TargetFramework) if Array.isEmpty testResults then let message = @@ -1606,13 +1614,7 @@ module Interactions = window.showWarningMessage (message) |> ignore TestRun.Output.appendWarningLine testRun message else - mergeResultsToExplorer - testRun - projectPath - projectRunRequest.TargetFramework - TrimMissing.Trim - runnableTests - testResults + mergeResultsToExplorer testRun TrimMissing.Trim runnableTests testResults } let runTestProject @@ -1776,44 +1778,25 @@ module Interactions = let private runTests_WithLanguageServer mergeTestResultsToExplorer + (rootTestCollection: TestItemCollection) (req: TestRunRequest) testRun - projectRunRequests = promise { try - let runnableTestsByProject = - projectRunRequests - |> Array.map (fun rr -> rr.ProjectPath, rr.Tests |> Array.collect TestItem.runnableChildren) - |> Map + let expectedToRun = + req.``include`` + |> Option.map Array.ofSeq + |> Option.defaultValue (rootTestCollection.TestItems()) + |> Array.collect TestItem.runnableChildren - let expectedTestsById = - runnableTestsByProject - |> Map.values - |> Seq.collect (Seq.map (fun t -> t.id, t)) - |> Map + let expectedTestsById = expectedToRun |> Array.map (fun t -> t.id, t) |> Map let mergeResults (shouldTrim: TrimMissing) (resultDtos: TestResultDTO array) = - let groups = - resultDtos - |> Array.groupBy (fun tr -> tr.TestItem.ProjectFilePath, tr.TestItem.TargetFramework) - - groups - |> Array.iter (fun ((projPath, targetFramework), results) -> - let expectedToRun = - runnableTestsByProject - |> Map.tryFind projPath - |> Option.defaultValue Array.empty - - let actuallyRan: TestResult array = results |> Array.map TestResult.ofTestResultDTO - - mergeTestResultsToExplorer - testRun - projPath - targetFramework - shouldTrim - expectedToRun - actuallyRan) + let actuallyRan: TestResult array = + resultDtos |> Array.map TestResult.ofTestResultDTO + + mergeTestResultsToExplorer testRun shouldTrim expectedToRun actuallyRan let showStarted (testItems: TestItemDTO array) = try @@ -1875,12 +1858,19 @@ module Interactions = | Some selectedCases when Seq.isEmpty selectedCases -> None, None | Some selectedCases -> let filter = - projectRunRequests - |> Array.collect (fun rr -> rr.Tests) + selectedCases + |> Array.ofSeq + |> Array.filter (fun t -> t.id |> TestItem.getFullName <> String.Empty) |> buildFilterExpression |> Some - let projectSubset = projectRunRequests |> Array.map (fun p -> p.ProjectPath) |> Some + let projectSubset = + selectedCases + |> Seq.map (TestItem.getId >> TestItem.getProjectPath) + |> Seq.distinct + |> Array.ofSeq + |> Some + filter, projectSubset logger.Debug($"Test Filter Expression: {filterExpression}") @@ -1923,7 +1913,6 @@ module Interactions = if testController.items.size < 1. then !! testRun.``end`` () else - let projectRunRequests = filtersToProjectRunRequests testController.items req let testItemFactory = TestItem.itemFactoryForController testController @@ -1933,7 +1922,6 @@ module Interactions = let runTestProject = runTestProject mergeTestResultsToExplorer makeTrxPath testRun _ct - let buildProject testRun projectRunRequest = promise { @@ -1952,6 +1940,7 @@ module Interactions = } promise { + let projectRunRequests = filtersToProjectRunRequests testController.items req projectRunRequests |> Array.collect (fun rr -> rr.Tests |> TestItem.runnableFromArray) @@ -1971,7 +1960,7 @@ module Interactions = () else - do! runTests_WithLanguageServer mergeTestResultsToExplorer req testRun projectRunRequests + do! runTests_WithLanguageServer mergeTestResultsToExplorer testController.items req testRun testRun.``end`` () } From 6713384a172dd4639a01451fa97838fbc6180eda Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Sun, 14 Sep 2025 15:56:56 -0500 Subject: [PATCH 24/32] Fix error when running parameterized test cases for xUnit or MSTest --- src/Components/TestExplorer.fs | 43 ++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 8616aa2e..45f08d01 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -276,7 +276,7 @@ module TestFrameworkId = None module TestItemDTO = - let getNormalizedFullName (dto: TestItemDTO) = + let getFullname_withNestedParamTests (dto: TestItemDTO) = match dto.ExecutorUri |> TestFrameworkId.tryFromExecutorUri with // NOTE: XUnit and MSTest don't include the theory case parameters in the FullyQualifiedName, but do include them in the DisplayName. // Thus we need to append the DisplayName to differentiate the test cases @@ -329,7 +329,7 @@ module TestResult = let ofTestResultDTO (testResultDto: TestResultDTO) : TestResult = let expected, actual = tryExtractExpectedAndActual testResultDto.ErrorMessage - { FullTestName = testResultDto.TestItem |> TestItemDTO.getNormalizedFullName + { FullTestName = testResultDto.TestItem |> TestItemDTO.getFullname_withNestedParamTests Outcome = testResultDto.Outcome |> TestResultOutcome.ofOutcomeDto Output = testResultDto.AdditionalOutput ErrorMessage = testResultDto.ErrorMessage @@ -979,7 +979,7 @@ module TestItem = let mapDtosForProject ((projectPath, targetFramework), flatTests) = let testDtoToNamedItem (dto: TestItemDTO) = {| Data = dto - FullName = dto |> TestItemDTO.getNormalizedFullName |} + FullName = dto |> TestItemDTO.getFullname_withNestedParamTests |} let namedHierarchies = flatTests |> Array.map testDtoToNamedItem |> TestName.inferHierarchy @@ -1426,7 +1426,42 @@ module Interactions = builder.ToString() let testToFilterExpression (test: TestItem) = - let fullTestName = TestItem.getFullName test.id + let isProbableParameterizedTest (test: TestItem) = + match test.parent with + | None -> false + | Some parent -> + let parentPlusParentheses = + RegularExpressions.Regex($"{parent.label |> RegularExpressions.Regex.Escape}\s*\(") + + parentPlusParentheses.IsMatch(test.label) + + let getFullNameOfParameterizedTest (test: TestItem) = + // NOTE: For xUnit and MSTest, we're nesting the the parameterized test cases under their method name, + // but the cannonical fully qualified test name doesn't reflect this nesting, so we have to account for the parent + // There might be a better way to handle this. Perhaps dynamically adding a cannonical unique test id field to TestItem + // (like with TestFramework). Adding this to runnable TestItems would reduce edge cases and special behavior for running individual tests + let maybeGrandParent = test.parent |> Option.bind (fun t -> t.parent) + + match maybeGrandParent with + | None -> TestItem.getFullName test.id + | Some grandParent -> + TestName.appendSegment + (TestItem.getFullName grandParent.id) + { Text = test.label + SeparatorBefore = string TestName.pathSeparator } + + let getFilterPath (test: TestItem) = + if + (test.TestFramework = TestFrameworkId.XUnit + || test.TestFramework = TestFrameworkId.MsTest) + && isProbableParameterizedTest test + then + getFullNameOfParameterizedTest test + else + TestItem.getFullName test.id + + + let fullTestName = getFilterPath test let escapedTestName = escapeFilterExpression fullTestName if escapedTestName.Contains(" ") && test.TestFramework = TestFrameworkId.NUnit then From 9c97dae6ab0e91edef4d2d247b5e991e3a943296 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Sun, 14 Sep 2025 16:23:45 -0500 Subject: [PATCH 25/32] Rediscover tests after a test run Test projects without code analysis updates (i.e C# projects or projects using Microsoft.Testing.Platform.VSTestBridge) will show new tests if you run a group of tests that changed, but those new tests will not have code locations. The lack of code locations in this limited case is very confusing and degrades the experience significantly. I looked at several options to fill in these locations. - VSTest does not return test locations on test runs, so mapping a location from results doesn't work - Running discovery before a test run caused test runs to feel sluggish. Especially when trying to run individual tests, which is a core workflow. I tried project-specific discovery and parallel discovery/test run, but still had an average ~1s delay. However, test discovery, even with thousands of tests, only takes a few seconds. That feels sluggish before the test run, but is inconsequential after the test run. The discovery is generally finished before the user has evaluated the test results. This pattern also has some precedent. Visual Studio will generally discover new tests in a group only after you run the group. Thanks for coming to my ted talk. --- src/Components/TestExplorer.fs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 45f08d01..ce9eabce 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1993,11 +1993,12 @@ module Interactions = successfullyBuiltRequests |> (Promise.executeWithMaxParallel maxParallelTestProjects runTestProject) - () + testRun.``end`` () else do! runTests_WithLanguageServer mergeTestResultsToExplorer testController.items req testRun + testRun.``end`` () + do! discoverTests_WithLangaugeServer testItemFactory testController.items tryGetLocation - testRun.``end`` () } |> (Promise.toThenable >> (!^)) From 7e104dff0fd034d590e671e88c878c0399cea81c Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 17 Sep 2025 11:32:04 -0500 Subject: [PATCH 26/32] Improve error reporting when the test run throws an exception --- src/Components/TestExplorer.fs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index ce9eabce..4b64d8fd 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1930,6 +1930,12 @@ module Interactions = TestRun.Output.appendWarningLine testRun message with ex -> logger.Debug("Test run failed with exception", ex) + TestRun.Output.appendErrorLine testRun $"The test run errored {Environment.NewLine}{string ex}" + + window.showErrorMessage ( + "Test run errored. See TestResults or Output > F# - Test Adapter for more info" + ) + |> ignore } let runHandler From 62d8f9b3dbea7e77ea750dfe19604115f1721521 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:23:01 -0500 Subject: [PATCH 27/32] Fix error when test names contain -- We currently store the test's FullyQualifiedName in the id and separate it from other id components with -- This led to incorrectly splitting the test name if it contained a -- It may be a good idea to dynamically add a field to TestItem for FullyQualifiedName similar to what was donen for TestFramework --- src/Components/TestExplorer.fs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 4b64d8fd..de488d26 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -766,8 +766,10 @@ module TestItem = let constructProjectRootId (projectPath: ProjectPath) : TestId = constructId projectPath "" let private componentizeId (testId: TestId) : (ProjectPath * FullTestName) = + // IMPORTANT: the fullname should be last and we should limit the number of substrings + // to prevent incorrently splitting tests names with -- in them let split = - testId.Split(separator = [| idSeparator |], options = StringSplitOptions.None) + testId.Split(separator = [| idSeparator |], count = 2, options = StringSplitOptions.None) (split.[0], split.[1]) From 359ea6b277c400a491b27b15ea96d8e65c68d340 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Thu, 18 Sep 2025 14:27:31 -0500 Subject: [PATCH 28/32] Fix spelling error --- src/Components/TestExplorer.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index de488d26..db9a8c34 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1737,7 +1737,7 @@ module Interactions = HasIncludeFilter = hasIncludeFilter Tests = replaceProjectRootIfPresent tests }) - let private discoverTests_WithLangaugeServer + let private discoverTests_WithLanguageServer testItemFactory (rootTestCollection: TestItemCollection) tryGetLocation @@ -2005,7 +2005,7 @@ module Interactions = else do! runTests_WithLanguageServer mergeTestResultsToExplorer testController.items req testRun testRun.``end`` () - do! discoverTests_WithLangaugeServer testItemFactory testController.items tryGetLocation + do! discoverTests_WithLanguageServer testItemFactory testController.items tryGetLocation } |> (Promise.toThenable >> (!^)) @@ -2160,7 +2160,7 @@ module Interactions = cancellationToken builtTestProjects else - do! discoverTests_WithLangaugeServer testItemFactory rootTestCollection tryGetLocation + do! discoverTests_WithLanguageServer testItemFactory rootTestCollection tryGetLocation } let tryMatchTestBySuffix (locationCache: CodeLocationCache) (testId: TestId) = From 326c472787b61f543bedd1d44f97be192d23f50e Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 23 Sep 2025 21:33:43 -0500 Subject: [PATCH 29/32] Replace custom Option extension methods with builtin methods tryFallback -> Option.orElseWith tryFallbackValue -> Option.orElse --- src/Components/TestExplorer.fs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index db9a8c34..9c27a1d5 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -85,15 +85,6 @@ module Option = option |> Option.iter f option - let tryFallback f opt = - match opt with - | Some _ -> opt - | None -> f () - - let tryFallbackValue fallbackOpt opt = - match opt with - | Some _ -> opt - | None -> fallbackOpt module CancellationToken = let mergeTokens (tokens: CancellationToken list) = @@ -322,7 +313,7 @@ module TestResult = Array.tryFind (fun (line: string) -> line.StartsWith(startsWith)) lines |> Option.map (fun line -> line.Replace(startsWith, "").TrimStart()) - tryFind "Expected:", tryFind "But was:" |> Option.tryFallbackValue (tryFind "Actual:") + tryFind "Expected:", tryFind "But was:" |> Option.orElse (tryFind "Actual:") expected, actual @@ -964,7 +955,7 @@ module TestItem = let codeLocation = namedNode.Data |> Option.bind tryDtoToLocation - |> Option.tryFallback (fun _ -> tryGetLocation id) + |> Option.orElseWith (fun _ -> tryGetLocation id) itemFactory { id = id @@ -2196,7 +2187,7 @@ module Interactions = let tryMatchDisplacedTest (testId: ResultBasedTestId) : TestItem option = displacedFragmentMapCache.TryGet(testId) - |> Option.tryFallback (fun () -> tryMatchTestBySuffix locationCache testId) + |> Option.orElseWith (fun () -> tryMatchTestBySuffix locationCache testId) |> Option.tee (fun matchedId -> displacedFragmentMapCache[testId] <- matchedId) |> Option.bind (fun matchedId -> TestItem.tryGetById matchedId testsFromCode) |> Option.tee (fun matchedTest -> From 941edaedfb1e32a046d56c5170886970f8bc9fce Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Tue, 23 Sep 2025 21:39:45 -0500 Subject: [PATCH 30/32] Error if unexpected TestOutcomeDTO value found. This should not happen. If it does, it means ionide and the language server are out of sync --- src/Components/TestExplorer.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 9c27a1d5..8e39fc98 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -237,6 +237,7 @@ module TestResultOutcome = | TestOutcomeDTO.Skipped -> TestResultOutcome.Skipped | TestOutcomeDTO.None -> TestResultOutcome.NotExecuted | TestOutcomeDTO.NotFound -> TestResultOutcome.NotExecuted + | _ -> failwith $"Unknown value for TestOutcomeDTO: {outcomeDto}. The language server may have changed its possible values." type TestFrameworkId = string From a4c788c954f3451fd27429b54554fd13dcb454ac Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 24 Sep 2025 08:53:30 -0500 Subject: [PATCH 31/32] Discover tests without building on initial workspace load If they've never built their solution, then we won't be able to load tests until they restore anyway. If they have built their solution, we probably don't need to build it again. It's much faster to discover tests without the build and, unlike clicking the refresh button, there is a much smaller chance they're looking for missing test or otherwise solving a weird explorer state where we'd want fresh build artifacts. --- src/Components/TestExplorer.fs | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 8e39fc98..486e0166 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -237,7 +237,9 @@ module TestResultOutcome = | TestOutcomeDTO.Skipped -> TestResultOutcome.Skipped | TestOutcomeDTO.None -> TestResultOutcome.NotExecuted | TestOutcomeDTO.NotFound -> TestResultOutcome.NotExecuted - | _ -> failwith $"Unknown value for TestOutcomeDTO: {outcomeDto}. The language server may have changed its possible values." + | _ -> + failwith + $"Unknown value for TestOutcomeDTO: {outcomeDto}. The language server may have changed its possible values." type TestFrameworkId = string @@ -1729,11 +1731,7 @@ module Interactions = HasIncludeFilter = hasIncludeFilter Tests = replaceProjectRootIfPresent tests }) - let private discoverTests_WithLanguageServer - testItemFactory - (rootTestCollection: TestItemCollection) - tryGetLocation - = + let discoverTests_WithLanguageServer testItemFactory (rootTestCollection: TestItemCollection) tryGetLocation = withProgress NoCancel <| fun p progressCancelToken -> promise { @@ -2319,16 +2317,19 @@ let activate (context: ExtensionContext) = let initialTests = trxTests workspaceProjects initialTests |> Array.iter testController.items.add - let cancellationTokenSource = vscode.CancellationTokenSource.Create() - // NOTE: Trx results can be partial if the last test run was filtered, so also queue a refresh to make sure we discover all tests - Interactions.refreshTestList - testItemFactory - testController.items - tryGetLocation - makeTrxPath - useLegacyDotnetCliIntegration - cancellationTokenSource.token - |> Promise.start + let cancellationTokenSource = vscode.CancellationTokenSource.Create() + // NOTE: Trx results can be partial if the last test run was filtered, so also queue a refresh to make sure we discover all tests + Interactions.refreshTestList + testItemFactory + testController.items + tryGetLocation + makeTrxPath + useLegacyDotnetCliIntegration + cancellationTokenSource.token + |> Promise.start + else + Interactions.discoverTests_WithLanguageServer testItemFactory testController.items tryGetLocation + |> Promise.start None) |> unbox From 6f10c227896249ef2132ef24fd6df5b4f102a905 Mon Sep 17 00:00:00 2001 From: farlee2121 <2847259+farlee2121@users.noreply.github.com> Date: Wed, 24 Sep 2025 15:09:41 -0500 Subject: [PATCH 32/32] Fix bug where tests with the same name could conflict between projects IcedTasks experienced this issue when all tests were run. Merging tests results used to be per-project, but now is not. So the test result venn requires project scope data --- src/Components/TestExplorer.fs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Components/TestExplorer.fs b/src/Components/TestExplorer.fs index 486e0166..8c6494e2 100644 --- a/src/Components/TestExplorer.fs +++ b/src/Components/TestExplorer.fs @@ -1550,8 +1550,10 @@ module Interactions = testResult.TargetFramework testResult.FullTestName - let treeItemComparable (t: TestItem) = TestItem.getFullName t.id - let resultComparable (r: TestResult) = r.FullTestName + let treeItemComparable (t: TestItem) = TestItem.getId t + + let resultComparable (r: TestResult) = + TestItem.constructId r.ProjectFilePath r.FullTestName let missing, expected, added = ArrayExt.venn treeItemComparable resultComparable expectedToRun testResults