Skip to content

Commit 3e4f122

Browse files
timcasselltimcassell
committed
Updated manual tests.
1 parent f588b6c commit 3e4f122

File tree

1 file changed

+49
-115
lines changed

1 file changed

+49
-115
lines changed

‎tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs‎

Lines changed: 49 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
using BenchmarkDotNet.Portability;
1313
using BenchmarkDotNet.Reports;
1414
using BenchmarkDotNet.Tests.XUnit;
15+
using BenchmarkDotNet.Toolchains;
16+
using BenchmarkDotNet.Toolchains.CsProj;
1517
using BenchmarkDotNet.Toolchains.InProcess.Emit;
1618
using Perfolizer;
1719
using Perfolizer.Horology;
@@ -26,11 +28,23 @@ namespace BenchmarkDotNet.IntegrationTests.ManualRunning
2628
{
2729
public class ExpectedBenchmarkResultsTests(ITestOutputHelper output) : BenchmarkTestExecutor(output)
2830
{
29-
// NativeAot takes a long time to build, so not including it in these tests.
30-
// We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains.
31-
3231
private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d);
3332

33+
// Visual Studio Test Explorer doesn't like to display IToolchain params separately, so use an enum instead.
34+
public enum ToolchainType
35+
{
36+
Default,
37+
InProcess
38+
}
39+
40+
private IToolchain GetToolchain(ToolchainType toolchain)
41+
=> toolchain switch
42+
{
43+
ToolchainType.Default => Job.Default.GetToolchain(),
44+
ToolchainType.InProcess => InProcessEmitToolchain.Instance,
45+
_ => throw new NotSupportedException()
46+
};
47+
3448
private static IEnumerable<Type> EmptyBenchmarkTypes() =>
3549
[
3650
typeof(EmptyVoid),
@@ -49,73 +63,32 @@ private static IEnumerable<Type> EmptyBenchmarkTypes() =>
4963
typeof(EmptyClass)
5064
];
5165

52-
public static IEnumerable<object[]> InProcessData()
66+
public static IEnumerable<object[]> GetEmptyArgs()
5367
{
5468
foreach (var type in EmptyBenchmarkTypes())
5569
{
56-
yield return new object[] { type };
57-
}
58-
}
59-
60-
public static IEnumerable<object[]> CoreData()
61-
{
62-
foreach (var type in EmptyBenchmarkTypes())
63-
{
64-
yield return new object[] { type, RuntimeMoniker.Net80 };
65-
yield return new object[] { type, RuntimeMoniker.Mono80 };
66-
}
67-
}
68-
69-
public static IEnumerable<object[]> FrameworkData()
70-
{
71-
foreach (var type in EmptyBenchmarkTypes())
72-
{
73-
yield return new object[] { type, RuntimeMoniker.Net462 };
74-
yield return new object[] { type, RuntimeMoniker.Mono };
70+
yield return new object[] { ToolchainType.Default, type };
71+
// InProcess overhead measurements are incorrect in Core. https://github.com/dotnet/runtime/issues/89685
72+
if (!RuntimeInformation.IsNetCore)
73+
{
74+
yield return new object[] { ToolchainType.InProcess, type };
75+
}
7576
}
7677
}
7778

7879
[Theory]
79-
[MemberData(nameof(InProcessData))]
80-
public void EmptyBenchmarkReportsZeroTimeAndAllocated_InProcess(Type benchmarkType)
81-
{
82-
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
83-
.AddJob(Job.Default
84-
.WithToolchain(InProcessEmitToolchain.Instance)
85-
// IL Emit has incorrect overhead measurement. https://github.com/dotnet/runtime/issues/89685
86-
// We multiply the threshold to account for it.
87-
), multiplyThresholdBy: RuntimeInformation.IsNetCore ? 3 : 1);
88-
}
89-
90-
[TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)]
91-
[MemberData(nameof(CoreData))]
92-
public void EmptyBenchmarkReportsZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
93-
{
94-
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
95-
.AddJob(Job.Default
96-
.WithRuntime(runtimeMoniker.GetRuntime())
97-
));
98-
}
99-
100-
[TheoryEnvSpecific("Can only run Full .NET Framework and Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)]
101-
[MemberData(nameof(FrameworkData))]
102-
public void EmptyBenchmarkReportsZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
103-
{
104-
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
105-
.AddJob(Job.Default
106-
.WithRuntime(runtimeMoniker.GetRuntime())
107-
));
108-
}
109-
110-
private void AssertZeroResults(Type benchmarkType, IConfig config, int multiplyThresholdBy = 1)
80+
[MemberData(nameof(GetEmptyArgs))]
81+
public void EmptyBenchmarkReportsZeroTimeAndAllocated(ToolchainType toolchain, Type benchmarkType)
11182
{
112-
var summary = CanExecute(benchmarkType, config
83+
var config = ManualConfig.CreateEmpty()
84+
.AddJob(Job.Default.WithToolchain(GetToolchain(toolchain)))
11385
.WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
114-
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
115-
);
86+
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)));
87+
88+
var summary = CanExecute(benchmarkType, config);
11689

11790
var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue;
118-
var threshold = new NumberValue(cpuResolution.Nanoseconds * multiplyThresholdBy).ToThreshold();
91+
var threshold = new NumberValue(cpuResolution.Nanoseconds).ToThreshold();
11992

12093
foreach (var report in summary.Reports)
12194
{
@@ -131,80 +104,41 @@ private void AssertZeroResults(Type benchmarkType, IConfig config, int multiplyT
131104

132105
private static IEnumerable<Type> NonEmptyBenchmarkTypes() =>
133106
[
134-
typeof(DifferentSizedStructs),
107+
// Structs even as large as Struct128 results in zero measurements on Zen 5, so the test will only pass on older CPU architectures.
108+
//typeof(DifferentSizedStructs),
135109
typeof(ActualWork)
136110
];
137111

138-
public static IEnumerable<object[]> NonEmptyInProcessData()
112+
public static IEnumerable<object[]> GetNonEmptyArgs()
139113
{
140114
foreach (var type in NonEmptyBenchmarkTypes())
141115
{
142-
yield return new object[] { type };
143-
}
144-
}
145-
146-
public static IEnumerable<object[]> NonEmptyCoreData()
147-
{
148-
foreach (var type in NonEmptyBenchmarkTypes())
149-
{
150-
yield return new object[] { type, RuntimeMoniker.Net80 };
151-
yield return new object[] { type, RuntimeMoniker.Mono80 };
152-
}
153-
}
154-
155-
public static IEnumerable<object[]> NonEmptyFrameworkData()
156-
{
157-
foreach (var type in NonEmptyBenchmarkTypes())
158-
{
159-
yield return new object[] { type, RuntimeMoniker.Net462 };
160-
yield return new object[] { type, RuntimeMoniker.Mono };
116+
// Framework is slightly less accurate than Core.
117+
yield return new object[] { ToolchainType.Default, type, RuntimeInformation.IsNetCore ? 0 : 1 };
118+
// InProcess overhead measurements are incorrect in Core. https://github.com/dotnet/runtime/issues/89685
119+
if (!RuntimeInformation.IsNetCore)
120+
{
121+
yield return new object[] { ToolchainType.InProcess, type, 1 };
122+
}
161123
}
162124
}
163125

164126
[Theory]
165-
[MemberData(nameof(NonEmptyInProcessData))]
166-
public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_InProcess(Type benchmarkType)
167-
{
168-
AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
169-
.AddJob(Job.Default
170-
.WithToolchain(InProcessEmitToolchain.Instance)
171-
// InProcess overhead measurements are incorrect, so we adjust the results to account for it. https://github.com/dotnet/runtime/issues/89685
172-
), subtractOverheadByClocks: RuntimeInformation.IsNetCore ? 3 : 1);
173-
}
174-
175-
[TheoryEnvSpecific("To not repeat tests in both Full .NET Framework and Core", EnvRequirement.DotNetCoreOnly)]
176-
[MemberData(nameof(NonEmptyCoreData))]
177-
public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
178-
{
179-
AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
180-
.AddJob(Job.Default
181-
.WithRuntime(runtimeMoniker.GetRuntime())
182-
));
183-
}
184-
185-
[TheoryEnvSpecific("Can only run Mono tests from Framework host", EnvRequirement.FullFrameworkOnly)]
186-
[MemberData(nameof(NonEmptyFrameworkData))]
187-
public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
188-
{
189-
AssertNonZeroResults(benchmarkType, ManualConfig.CreateEmpty()
190-
.AddJob(Job.Default
191-
.WithRuntime(runtimeMoniker.GetRuntime())
192-
));
193-
}
194-
195-
private void AssertNonZeroResults(Type benchmarkType, IConfig config, int subtractOverheadByClocks = 0)
127+
[MemberData(nameof(GetNonEmptyArgs))]
128+
public void NonEmptyBenchmarkReportsNonZeroTimeAndZeroAllocated(ToolchainType toolchain, Type benchmarkType, int subtractOverheadByClocks)
196129
{
197-
var summary = CanExecute(benchmarkType, config
130+
var config = ManualConfig.CreateEmpty()
131+
.AddJob(Job.Default.WithToolchain(GetToolchain(toolchain)))
198132
.WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
199-
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
200-
);
133+
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)));
134+
135+
var summary = CanExecute(benchmarkType, config);
201136

202137
var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue;
203138
// Modern cpus can execute multiple instructions per clock cycle,
204139
// resulting in measurements greater than 0 but less than 1 clock cycle.
205140
// (example: Intel Core i9-9880H CPU 2.30GHz reports 0.2852 ns for `_field++;`)
206141
var threshold = new NumberValue(cpuResolution.Nanoseconds / 4).ToThreshold();
207-
// InProcess overhead measurements are incorrect, so we adjust the results to account for it. https://github.com/dotnet/runtime/issues/89685
208142
var overheadSubtraction = cpuResolution.Nanoseconds * subtractOverheadByClocks;
209143

210144
foreach (var report in summary.Reports)

0 commit comments

Comments
 (0)