Skip to content

Adding tests for export of csv and csv Measurements #2387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Adding tests for export of csv and csv Measurements #2387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9971069
Adding tests for export of csv and csv Measurements
Nepp3r Jul 31, 2023
b6a8ab0
Adding fixed affinity for tests
Nepp3r Aug 5, 2023
5f2fc7f
Some smaller design fixes
Nepp3r Aug 5, 2023
65163ce
Removing not-actual comment
Nepp3r Aug 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions src/BenchmarkDotNet/Environments/EnvironmentResolver.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Diagnostics.CodeAnalysis;
using BenchmarkDotNet.Characteristics;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Jobs;
Expand All @@ -9,10 +10,15 @@ namespace BenchmarkDotNet.Environments
{
public class EnvironmentResolver : Resolver
{
public const int DefaultUnrollFactorForThroughput = 16;

public static readonly IResolver Instance = new CompositeResolver(new EnvironmentResolver(), GcResolver.Instance);

public const int DefaultUnrollFactorForThroughput = 16;

internal static readonly EnvironmentResolver Default = new EnvironmentResolver();

public static readonly IResolver Instance = new CompositeResolver(Default, GcResolver.Instance);
internal new void Register<[DynamicallyAccessedMembers(CharacteristicObject.CharacteristicMemberTypes)] T>(Characteristic<T> characteristic, Func<T> resolver)
{
base.Register(characteristic, resolver);
}
private EnvironmentResolver()
{
Register(EnvironmentMode.PlatformCharacteristic, RuntimeInformation.GetCurrentPlatform);
Expand Down
19 changes: 13 additions & 6 deletions tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Exporters.Csv;
using BenchmarkDotNet.Exporters.Json;
using BenchmarkDotNet.Exporters.Xml;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Tests.Builders;
using BenchmarkDotNet.Tests.Mocks;
Expand Down Expand Up @@ -43,14 +48,15 @@ public CommonExporterVerifyTests()
public Task Exporters(string cultureInfoName)
{
var cultureInfo = CultureInfos[cultureInfoName];
Thread.CurrentThread.CurrentCulture = cultureInfo;

Thread.CurrentThread.CurrentCulture = cultureInfo;
EnvironmentResolver.Default.Register(EnvironmentMode.AffinityCharacteristic, () => new IntPtr(0xFF));

var logger = new AccumulationLogger();

var exporters = GetExporters();
foreach (var exporter in exporters)
{
PrintTitle(logger, exporter);
PrintTitle(logger, exporter);
exporter.ExportToLog(MockFactory.CreateSummary(config.WithCultureInfo(cultureInfo)), logger);
}

Expand All @@ -75,7 +81,6 @@ private static string GetName(CultureInfo cultureInfo)

private static IEnumerable<IExporter> GetExporters()
{
//todo add CsvExporter and CsvMeasurementsExporter (need to mock RuntimeInformation)
yield return AsciiDocExporter.Default;
yield return HtmlExporter.Default;
yield return JsonExporter.Brief;
Expand All @@ -92,8 +97,10 @@ private static IEnumerable<IExporter> GetExporters()
yield return XmlExporter.BriefCompressed;
yield return XmlExporter.Full;
yield return XmlExporter.FullCompressed;
}

yield return CsvExporter.Default;
yield return CsvMeasurementsExporter.Default;
}

private static readonly IConfig config = ManualConfig.Create(DefaultConfig.Instance)
.AddColumn(StatisticColumn.Mean)
.AddColumn(StatisticColumn.StdDev)
Expand Down
12 changes: 12 additions & 0 deletions ....Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -812,3 +812,15 @@ XmlExporter-full
XmlExporter-full-compressed
############################################
<?xml version="1.0" encoding="utf-8"?><Summary><Title>MockSummary</Title><HostEnvironmentInfo><BenchmarkDotNetCaption>BenchmarkDotNet</BenchmarkDotNetCaption><BenchmarkDotNetVersion>0.10.x-mock</BenchmarkDotNetVersion><OsVersion>Microsoft Windows NT 10.0.x.mock</OsVersion><ProcessorName>MockIntel Core i7-6700HQ CPU 2.60GHz</ProcessorName><PhysicalProcessorCount>1</PhysicalProcessorCount><PhysicalCoreCount>4</PhysicalCoreCount><LogicalCoreCount>8</LogicalCoreCount><RuntimeVersion>Clr 4.0.x.mock</RuntimeVersion><Architecture>64mock</Architecture><HasAttachedDebugger>False</HasAttachedDebugger><HasRyuJit>True</HasRyuJit><Configuration>CONFIGURATION</Configuration><ChronometerFrequency><Hertz>2531248</Hertz></ChronometerFrequency><HardwareTimerKind>Tsc</HardwareTimerKind></HostEnvironmentInfo><Benchmarks><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Foo</Method><MethodTitle>Foo</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Bar</Method><MethodTitle>Bar</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase></Benchmarks></Summary>
############################################
CsvExporter
############################################
Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,P67
Foo,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
Bar,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
############################################
CsvMeasurementsExporter
############################################
Target,Target_Namespace,Target_Type,Target_Method,Job_Id,Job_AnalyzeLaunchVariance,Job_EvaluateOverhead,Job_MaxAbsoluteError,Job_MaxRelativeError,Job_MinInvokeCount,Job_MinIterationTime,Job_OutlierMode,Job_Affinity,Job_EnvironmentVariables,Job_Jit,Job_LargeAddressAware,Job_Platform,Job_PowerPlanMode,Job_Runtime,Job_AllowVeryLargeObjects,Job_Concurrent,Job_CpuGroups,Job_Force,Job_HeapAffinitizeMask,Job_HeapCount,Job_NoAffinitize,Job_RetainVm,Job_Server,Job_Arguments,Job_BuildConfiguration,Job_Clock,Job_EngineFactory,Job_NuGetReferences,Job_Toolchain,Job_IsMutator,Job_InvocationCount,Job_IterationCount,Job_IterationTime,Job_LaunchCount,Job_MaxIterationCount,Job_MaxWarmupIterationCount,Job_MemoryRandomization,Job_MinIterationCount,Job_MinWarmupIterationCount,Job_RunStrategy,Job_UnrollFactor,Job_WarmupCount,Job_Display,Params,Measurement_LaunchIndex,Measurement_IterationMode,Measurement_IterationStage,Measurement_IterationIndex,Measurement_Nanoseconds,Measurement_Operations,Measurement_Value,Gen_0,Gen_1,Gen_2,Allocated_Bytes
MockBenchmarkClass.Foo,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Foo,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?
MockBenchmarkClass.Bar,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Bar,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?
12 changes: 12 additions & 0 deletions ...tNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -812,3 +812,15 @@ XmlExporter-full
XmlExporter-full-compressed
############################################
<?xml version="1.0" encoding="utf-8"?><Summary><Title>MockSummary</Title><HostEnvironmentInfo><BenchmarkDotNetCaption>BenchmarkDotNet</BenchmarkDotNetCaption><BenchmarkDotNetVersion>0.10.x-mock</BenchmarkDotNetVersion><OsVersion>Microsoft Windows NT 10.0.x.mock</OsVersion><ProcessorName>MockIntel Core i7-6700HQ CPU 2.60GHz</ProcessorName><PhysicalProcessorCount>1</PhysicalProcessorCount><PhysicalCoreCount>4</PhysicalCoreCount><LogicalCoreCount>8</LogicalCoreCount><RuntimeVersion>Clr 4.0.x.mock</RuntimeVersion><Architecture>64mock</Architecture><HasAttachedDebugger>False</HasAttachedDebugger><HasRyuJit>True</HasRyuJit><Configuration>CONFIGURATION</Configuration><ChronometerFrequency><Hertz>2531248</Hertz></ChronometerFrequency><HardwareTimerKind>Tsc</HardwareTimerKind></HostEnvironmentInfo><Benchmarks><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Foo</Method><MethodTitle>Foo</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase><BenchmarkCase><DisplayInfo>MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)</DisplayInfo><Namespace>BenchmarkDotNet.Tests.Mocks</Namespace><Type>MockBenchmarkClass</Type><Method>Bar</Method><MethodTitle>Bar</MethodTitle><Statistics><OriginalValues><Item>1</Item></OriginalValues><N>1</N><Min>1</Min><LowerFence>1</LowerFence><Q1>1</Q1><Median>1</Median><Mean>1</Mean><Q3>1</Q3><UpperFence>1</UpperFence><Max>1</Max><InterquartileRange>0</InterquartileRange><StandardError>0</StandardError><Variance>0</Variance><StandardDeviation>0</StandardDeviation><Skewness>NaN</Skewness><Kurtosis>NaN</Kurtosis><ConfidenceInterval><N>1</N><Mean>1</Mean><StandardError>0</StandardError><Level>L999</Level><Margin>NaN</Margin><Lower>NaN</Lower><Upper>NaN</Upper></ConfidenceInterval><Percentiles><P0>1</P0><P25>1</P25><P50>1</P50><P67>1</P67><P80>1</P80><P85>1</P85><P90>1</P90><P95>1</P95><P100>1</P100></Percentiles></Statistics><Memory><Gen0Collections>0</Gen0Collections><Gen1Collections>0</Gen1Collections><Gen2Collections>0</Gen2Collections><TotalOperations>0</TotalOperations><BytesAllocatedPerOperation /></Memory><Measurements><Measurement><IterationMode>Workload</IterationMode><IterationStage>Result</IterationStage><LaunchIndex>1</LaunchIndex><IterationIndex>1</IterationIndex><Operations>1</Operations><Nanoseconds>1</Nanoseconds></Measurement></Measurements></BenchmarkCase></Benchmarks></Summary>
############################################
CsvExporter
############################################
Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,P67
Foo,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
Bar,LongRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,16,15,1.000 ns,NA,0.000 ns,1.000 ns
############################################
CsvMeasurementsExporter
############################################
Target,Target_Namespace,Target_Type,Target_Method,Job_Id,Job_AnalyzeLaunchVariance,Job_EvaluateOverhead,Job_MaxAbsoluteError,Job_MaxRelativeError,Job_MinInvokeCount,Job_MinIterationTime,Job_OutlierMode,Job_Affinity,Job_EnvironmentVariables,Job_Jit,Job_LargeAddressAware,Job_Platform,Job_PowerPlanMode,Job_Runtime,Job_AllowVeryLargeObjects,Job_Concurrent,Job_CpuGroups,Job_Force,Job_HeapAffinitizeMask,Job_HeapCount,Job_NoAffinitize,Job_RetainVm,Job_Server,Job_Arguments,Job_BuildConfiguration,Job_Clock,Job_EngineFactory,Job_NuGetReferences,Job_Toolchain,Job_IsMutator,Job_InvocationCount,Job_IterationCount,Job_IterationTime,Job_LaunchCount,Job_MaxIterationCount,Job_MaxWarmupIterationCount,Job_MemoryRandomization,Job_MinIterationCount,Job_MinWarmupIterationCount,Job_RunStrategy,Job_UnrollFactor,Job_WarmupCount,Job_Display,Params,Measurement_LaunchIndex,Measurement_IterationMode,Measurement_IterationStage,Measurement_IterationIndex,Measurement_Nanoseconds,Measurement_Operations,Measurement_Value,Gen_0,Gen_1,Gen_2,Allocated_Bytes
MockBenchmarkClass.Foo,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Foo,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?
MockBenchmarkClass.Bar,BenchmarkDotNet.Tests.Mocks,MockBenchmarkClass,Bar,LongRun,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,100,Default,3,Default,Default,Default,Default,Default,Default,Default,15,"LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)",,1,Workload,Result,1,1,1,1,0,0,0,?
Loading