diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml index 8dac83d30a2..17be7e9ab35 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml @@ -12,6 +12,10 @@ coreruns: environment_variables: DOTNET_GCName: clrgc.dll +iterations: + gcperfsim: 1 + microbenchmarks: 1 + trace_configuration_type: gc # Choose between: none, gc, verbose, cpu, cpu_managed, threadtime, join. # Optional fields: the contents of both the symbol_path and the source_path will be copied over to the output path. diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index 85c8821a8ec..5f0309b21dd 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -31,5 +31,38 @@ public static double StandardDeviation(this IEnumerable doubleList) double sumOfDerivationAverage = sumOfDerivation / (doubleList.Count() - 1); return Math.Sqrt(sumOfDerivationAverage - (average * average)); } + + public static IEnumerable RemoveOutliers(IEnumerable collection) + { + List validCollection = new(); + if (!collection.Any()) + { + return Array.Empty(); + } + foreach (double value in collection) + { + if (!double.IsNaN(value) && !double.IsInfinity(value)) + { + validCollection.Add(value); + } + } + if (validCollection.Count == 0) + { + return Array.Empty(); + } + // Calculate Q1 (25th percentile) and Q3 (75th percentile) + double q1 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.25); + double q3 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.75); + + // Calculate IQR (Interquartile Range) + double iqr = q3 - q1; + + // Calculate bounds: [Q1 - 1.5*IQR, Q3 + 1.5*IQR] + double lowerBound = q1 - 1.5 * iqr; + double upperBound = q3 + 1.5 * iqr; + + // Filter out outliers + return GoodLinq.Where(validCollection, x => x >= lowerBound && x <= upperBound); + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs similarity index 53% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs index 2af39abd257..a64f11000a4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs @@ -1,103 +1,10 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.GCPerfSim; -using Newtonsoft.Json; - -namespace GC.Infrastructure.Core.Analysis +namespace GC.Infrastructure.Core.Analysis { - public sealed class MicrobenchmarkResult - { - public Statistics Statistics { get; set; } - - [JsonIgnore] - public GCProcessData? GCData { get; set; } - - public ResultItem ResultItem { get; set; } - - [JsonIgnore] - public CPUProcessData? CPUData { get; set; } - public Run Parent { get; set; } - public string MicrobenchmarkName { get; set; } - public Dictionary OtherMetrics { get; set; } = new(); - - private static readonly IReadOnlyDictionary> _customStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { "number of iterations", (Statistics stats) => stats.N }, - { "min", (Statistics stats) => stats.Min }, - { "max", (Statistics stats) => stats.Max }, - { "median", (Statistics stats) => stats.Median }, - { "q1", (Statistics stats) => stats.Q1 }, - { "q3", (Statistics stats) => stats.Q3 }, - { "variance", (Statistics stats) => stats.Variance }, - { "standard deviation", (Statistics stats) => stats.StandardDeviation }, - { "skewness", (Statistics stats) => stats.Skewness }, - { "kurtosis", (Statistics stats) => stats.Kurtosis }, - { "standard error", (Statistics stats) => stats.StandardError }, - { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, - }; - - public static double? LookupStatisticsCalculation(string columnName, MicrobenchmarkResult result) - { - if (string.IsNullOrEmpty(columnName)) - { - return null; - } - - if (!_customStatisticsCalculationMap.TryGetValue(columnName, out var val)) - { - return null; - } - - else - { - return val.Invoke(result.Statistics); - } - } - } - - public sealed class Benchmark - { - public string DisplayInfo { get; set; } - public string Namespace { get; set; } - public string Type { get; set; } - public string Method { get; set; } - public string MethodTitle { get; set; } - public string Parameters { get; set; } - public string FullName { get; set; } - public Statistics Statistics { get; set; } - public Memory Memory { get; set; } - public List Measurements { get; set; } - public List Metrics { get; set; } - } - public sealed class ChronometerFrequency { public int Hertz { get; set; } } - public sealed class ConfidenceInterval - { - public int N { get; set; } - public double? Mean { get; set; } - public double? StandardError { get; set; } - public int? Level { get; set; } - public double? Margin { get; set; } - public double? Lower { get; set; } - public double? Upper { get; set; } - } - - public sealed class Descriptor - { - public string Id { get; set; } - public string DisplayName { get; set; } - public string Legend { get; set; } - public string NumberFormat { get; set; } - public int UnitType { get; set; } - public string Unit { get; set; } - public bool TheGreaterTheBetter { get; set; } - public int PriorityInCategory { get; set; } - } - public sealed class HostEnvironmentInfo { public string BenchmarkDotNetCaption { get; set; } @@ -117,6 +24,16 @@ public sealed class HostEnvironmentInfo public string HardwareTimerKind { get; set; } } + + public sealed class Memory + { + public int Gen0Collections { get; set; } + public int Gen1Collections { get; set; } + public int Gen2Collections { get; set; } + public int TotalOperations { get; set; } + public long BytesAllocatedPerOperation { get; set; } + } + public sealed class Measurement { public string IterationMode { get; set; } @@ -127,19 +44,27 @@ public sealed class Measurement public long Nanoseconds { get; set; } } - public sealed class Memory + public sealed class Descriptor { - public int Gen0Collections { get; set; } - public int Gen1Collections { get; set; } - public int Gen2Collections { get; set; } - public int TotalOperations { get; set; } - public long BytesAllocatedPerOperation { get; set; } + public string Id { get; set; } + public string DisplayName { get; set; } + public string Legend { get; set; } + public string NumberFormat { get; set; } + public int UnitType { get; set; } + public string Unit { get; set; } + public bool TheGreaterTheBetter { get; set; } + public int PriorityInCategory { get; set; } } - public sealed class Metric + public sealed class ConfidenceInterval { - public double Value { get; set; } - public Descriptor Descriptor { get; set; } + public int N { get; set; } + public double Mean { get; set; } + public double StandardError { get; set; } + public int Level { get; set; } + public double Margin { get; set; } + public double Lower { get; set; } + public double Upper { get; set; } } public sealed class Percentiles @@ -155,35 +80,56 @@ public sealed class Percentiles public double P100 { get; set; } } - public sealed class MicrobenchmarkResults - { - public string Title { get; set; } - public HostEnvironmentInfo HostEnvironmentInfo { get; set; } - public List Benchmarks { get; set; } - } - public sealed class Statistics { public List OriginalValues { get; set; } public int N { get; set; } - public double? Min { get; set; } - public double? LowerFence { get; set; } - public double? Q1 { get; set; } - public double? Median { get; set; } - public double? Mean { get; set; } - public double? Q3 { get; set; } - public double? UpperFence { get; set; } - public double? Max { get; set; } - public double? InterquartileRange { get; set; } - public List LowerOutliers { get; set; } + public double Min { get; set; } + public double LowerFence { get; set; } + public double Q1 { get; set; } + public double Median { get; set; } + public double Mean { get; set; } + public double Q3 { get; set; } + public double UpperFence { get; set; } + public double Max { get; set; } + public double InterquartileRange { get; set; } + public List LowerOutliers { get; set; } public List UpperOutliers { get; set; } - public List AllOutliers { get; set; } - public double? StandardError { get; set; } - public double? Variance { get; set; } - public double? StandardDeviation { get; set; } - public double? Skewness { get; set; } - public double? Kurtosis { get; set; } + public List AllOutliers { get; set; } + public double StandardError { get; set; } + public double Variance { get; set; } + public double StandardDeviation { get; set; } + public double Skewness { get; set; } + public double Kurtosis { get; set; } public ConfidenceInterval? ConfidenceInterval { get; set; } public Percentiles Percentiles { get; set; } } -} \ No newline at end of file + + public sealed class Metric + { + public double Value { get; set; } + public Descriptor Descriptor { get; set; } + } + + public sealed class Benchmark + { + public string DisplayInfo { get; set; } + public string Namespace { get; set; } + public string Type { get; set; } + public string Method { get; set; } + public string MethodTitle { get; set; } + public string Parameters { get; set; } + public string FullName { get; set; } + public Statistics Statistics { get; set; } + public Memory Memory { get; set; } + public List Measurements { get; set; } + public List Metrics { get; set; } + } + + public sealed class BdnJsonResult + { + public string Title { get; set; } + public HostEnvironmentInfo HostEnvironmentInfo { get; set; } + public List Benchmarks { get; set; } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs new file mode 100644 index 00000000000..3715fb5657a --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs @@ -0,0 +1,8 @@ +namespace GC.Infrastructure.Core.Analysis +{ + public static class GCTraceMetricComparison + { + public static GCTraceMetricComparisonResult CompareGCTraceMetric(IEnumerable baselines, IEnumerable comparands,string nameOfMetric) + => new GCTraceMetricComparisonResult(baselines, comparands, nameOfMetric); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs new file mode 100644 index 00000000000..9f749a6a0c7 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -0,0 +1,109 @@ +using API = GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Analysis.GC; +using System.Reflection; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetricComparisonResult + { + public GCTraceMetricComparisonResult(IEnumerable baselines, IEnumerable comparands, string metricName) + { + RunName = baselines.FirstOrDefault()?.RunName; + Key = $"{baselines.FirstOrDefault()?.ConfigurationName}_{RunName}"; + + MetricName = metricName; + PropertyInfo pInfo = typeof(GCTraceMetrics).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + + // Property found on the GCTraceMetrics. + if (pInfo != null) + { + OriginalBaselineMetricCollection = baselines.Select(baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = comparands.Select(comparand => (double)pInfo.GetValue(comparand)); + } + + // If property isn't found on the GCTraceMetrics, look in GCStats. + // TODO: Add the case where we look into the map. + else + { + pInfo = typeof(GCStats).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + if (pInfo == null) + { + FieldInfo fieldInfo = typeof(GCStats).GetField(metricName, BindingFlags.Instance | BindingFlags.Public); + if (fieldInfo == null) + { + // Out of luck! + OriginalBaselineMetricCollection = Array.Empty(); + OriginalComparandMetricCollection = Array.Empty(); + OutliersFreeBaselineMetricCollection = Array.Empty(); + OutliersFreeComparandMetricCollection = Array.Empty(); + AveragedBaselineMetric = double.NaN; + AveragedComparandMetric = double.NaN; + return; + } + + else + { + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(fieldInfo.Name)) + .Select(baseline => baseline.StatsData[fieldInfo.Name]); + + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(fieldInfo.Name)) + .Select(comparand => comparand.StatsData[fieldInfo.Name]); + } + } + + else + { + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(pInfo.Name)) + .Select(baseline => baseline.StatsData[pInfo.Name]); + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(pInfo.Name)) + .Select(comparand => comparand.StatsData[pInfo.Name]); + } + } + + // Filter out outliers using IQR method + OutliersFreeBaselineMetricCollection = API.Statistics.RemoveOutliers(OriginalBaselineMetricCollection); + OutliersFreeComparandMetricCollection = API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); + + // Calculate averaged metrics + AveragedBaselineMetric = API.GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r); + AveragedComparandMetric = API.GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r); + } + + public string RunName { get; } + public string Key { get; } + public string MetricName { get; } + public IEnumerable OriginalBaselineMetricCollection { get; } + public IEnumerable OriginalComparandMetricCollection { get; } + public IEnumerable OutliersFreeBaselineMetricCollection { get; } + public IEnumerable OutliersFreeComparandMetricCollection { get; } + + public double AveragedBaselineMetric { get; } + public double AveragedComparandMetric { get; } + public double Delta => AveragedComparandMetric - AveragedBaselineMetric; + public double PercentageDelta + { + get + { + if (AveragedBaselineMetric == 0) + { + if (AveragedComparandMetric == 0) + { + return 0; + } + else + { + return double.NaN; + } + } + else + { + return Delta / AveragedBaselineMetric * 100.0; + } + } + } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs new file mode 100644 index 00000000000..d88777bc010 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -0,0 +1,149 @@ +using GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetrics + { + public static GCTraceMetrics GetNullItem(string runName, string corerun) => + new GCTraceMetrics(runName, corerun); + + private GCTraceMetrics(string runName, string corerun) + { + ConfigurationName = corerun; + RunName = runName; + PctTimePausedInGC = double.NaN; + FirstToLastGCSeconds = double.NaN; + HeapSizeBeforeMB_Mean = double.NaN; + HeapSizeAfter_Mean = double.NaN; + TotalCommittedInUse = double.NaN; + TotalBookkeepingCommitted = double.NaN; + TotalCommittedInGlobalDecommit = double.NaN; + TotalCommittedInFree = double.NaN; + TotalCommittedInGlobalFree = double.NaN; + PauseDurationMSec_95PWhereIsGen0 = double.NaN; + PauseDurationMSec_95PWhereIsGen1 = double.NaN; + PauseDurationMSec_95PWhereIsBackground = double.NaN; + PauseDurationMSec_MeanWhereIsBackground = double.NaN; + PauseDurationMSec_95PWhereIsBlockingGen2 = double.NaN; + PauseDurationMSec_MeanWhereIsBlockingGen2 = double.NaN; + CountIsBlockingGen2 = double.NaN; + PauseDurationMSec_SumWhereIsGen1 = double.NaN; + PauseDurationMSec_MeanWhereIsEphemeral = double.NaN; + PromotedMB_MeanWhereIsGen1 = double.NaN; + CountIsGen1 = double.NaN; + CountIsGen0 = double.NaN; + HeapCount = double.NaN; + PauseDurationMSec_Sum = double.NaN; + TotalAllocatedMB = double.NaN; + TotalNumberGCs = double.NaN; + Speed_MBPerMSec = double.NaN; + ExecutionTimeMSec = double.NaN; + } + + public GCTraceMetrics(GCProcessData processData, string runName, string configurationName) + { + RunName = runName; + ConfigurationName = configurationName; + ExecutionTimeMSec = processData.DurationMSec; + + PctTimePausedInGC = processData.Stats.GetGCPauseTimePercentage(); + FirstToLastGCSeconds = (processData.GCs.Last().StartRelativeMSec - processData.GCs.First().StartRelativeMSec) / 1000; + HeapSizeAfter_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeAfterMB)); + HeapSizeBeforeMB_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeBeforeMB)); + + TotalCommittedInUse = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInUse)); + TotalBookkeepingCommitted = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalBookkeepingCommitted)); + TotalCommittedInGlobalDecommit = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalDecommit)); + TotalCommittedInFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInFree)); + TotalCommittedInGlobalFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalFree)); + + var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var property in properties) + { + if (property.PropertyType != typeof(double) && property.PropertyType != typeof(int)) + { + continue; + } + + string propertyName = property.Name; + object? value = property.GetValue(processData.Stats); + double propertyValue = value != null ? Convert.ToDouble(value) : double.NaN; + StatsData[propertyName] = propertyValue; + } + + var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var field in fields) + { + if (field.FieldType != typeof(double) && field.FieldType != typeof(int)) + { + continue; + } + + string name = field.Name; + object? value = field.GetValue(processData.Stats); + double doubleValue = value != null ? Convert.ToDouble(value) : double.NaN; + StatsData[name] = doubleValue; + } + + // 95P + PauseDurationMSec_95PWhereIsGen0 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 0)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_95PWhereIsGen1 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)), 0.95); + + PauseDurationMSec_95PWhereIsBackground = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBackground = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), (p => p)); + + PauseDurationMSec_95PWhereIsBlockingGen2 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBlockingGen2 = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), (p => p)); + + CountIsBlockingGen2 = processData.GCs.Count(gc => gc.Generation == 2 && gc.Type != GCType.BackgroundGC); + + HeapCount = processData.Stats.HeapCount; + TotalNumberGCs = processData.Stats.Count; + TotalAllocatedMB = processData.Stats.TotalAllocatedMB; + + Speed_MBPerMSec = processData.Stats.TotalPromotedMB / processData.Stats.TotalPauseTimeMSec; + + PauseDurationMSec_MeanWhereIsEphemeral = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1 || gc.Generation == 0)), (gc => gc.PauseDurationMSec)); + PauseDurationMSec_SumWhereIsGen1 = + GoodLinq.Sum(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)); + PromotedMB_MeanWhereIsGen1 = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PromotedMB)); + PauseDurationMSec_Sum = GoodLinq.Sum(processData.GCs, (gc => gc.PauseDurationMSec)); + CountIsGen1 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 1).Count; + CountIsGen0 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 0).Count; + } + + public double PctTimePausedInGC { get; } + public double FirstToLastGCSeconds { get; } + public double HeapSizeBeforeMB_Mean { get; } + public double HeapSizeAfter_Mean { get; } + public double TotalCommittedInUse { get; set; } + public double TotalCommittedInGlobalDecommit { get; set; } + public double TotalCommittedInFree { get; set; } + public double TotalCommittedInGlobalFree { get; set; } + public double TotalBookkeepingCommitted { get; set; } + public double PauseDurationMSec_95PWhereIsGen0 { get; } + public double PauseDurationMSec_95PWhereIsGen1 { get; } + public double PauseDurationMSec_95PWhereIsBackground { get; } + public double PauseDurationMSec_MeanWhereIsBackground { get; } + public double PauseDurationMSec_95PWhereIsBlockingGen2 { get; } + public double PauseDurationMSec_MeanWhereIsBlockingGen2 { get; } + public double CountIsBlockingGen2 { get; } + public double PauseDurationMSec_SumWhereIsGen1 { get; } + public double PauseDurationMSec_MeanWhereIsEphemeral { get; } + public double PromotedMB_MeanWhereIsGen1 { get; } + public double CountIsGen1 { get; } + public double CountIsGen0 { get; } + public double HeapCount { get; } + public double PauseDurationMSec_Sum { get; } + public double TotalAllocatedMB { get; set; } + public double TotalNumberGCs { get; } + public double Speed_MBPerMSec { get; } + public string RunName { get; } + public string ConfigurationName { get; } + public double ExecutionTimeMSec { get; } + public Dictionary StatsData { get; } = new(); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 60150f4d4c9..c2f5c40336b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -1,60 +1,170 @@ -using GC.Infrastructure.Core.Presentation.GCPerfSim; +using API = GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { // Per Microbenchmark result. public sealed class MicrobenchmarkComparisonResult { + public static readonly string[] RequiredMetrics = new string[] + { + "PctTimePausedInGC", + "ExecutionTimeMSec", + "PauseDurationMSec_MeanWhereIsEphemeral", + "PauseDurationMSec_MeanWhereIsBackground", + "PauseDurationMSec_MeanWhereIsBlockingGen2" + }; + public MicrobenchmarkComparisonResult() { } - public MicrobenchmarkComparisonResult(MicrobenchmarkResult baseline, MicrobenchmarkResult comparand) + public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool includeTraces = true) { - Baseline = baseline; - Comparand = comparand; - var result = new ResultItemComparison(baseline.ResultItem, comparand.ResultItem); ComparisonResults = new(); - ComparisonResults.Add(result.GetComparison("PctTimePausedInGC")); - ComparisonResults.Add(result.GetComparison("ExecutionTimeMSec")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsEphemeral")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBackground")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBlockingGen2")); + if (includeTraces) + { + var baselineGCTraceMetricsCollection = baselines + .Where(baseline => baseline != null) + .Where(baseline => baseline.GCTraceMetrics != null) + .Select(baseline => baseline.GCTraceMetrics) + .ToArray(); + + var comparandGCTraceMetricsCollection = comparands + .Where(comparand => comparand != null) + .Where(comparand => comparand.GCTraceMetrics != null) + .Select(comparand => comparand.GCTraceMetrics) + .ToArray(); + + if (baselineGCTraceMetricsCollection.Length > 0 && comparandGCTraceMetricsCollection.Length > 0) + { + foreach (var metricName in RequiredMetrics) + { + ComparisonResults.Add( + GCTraceMetricComparison.CompareGCTraceMetric( + baselineGCTraceMetricsCollection!, comparandGCTraceMetricsCollection!, metricName)); + } + + } + } + + var firstBaseline = baselines?.FirstOrDefault(); + var firstComparand = comparands?.FirstOrDefault(); + + BaselineRunName = firstBaseline?.Parent?.Name ?? string.Empty; + ComparandRunName = firstComparand?.Parent?.Name ?? string.Empty; + MicrobenchmarkName = firstBaseline?.MicrobenchmarkName ?? string.Empty; + + Baselines = baselines ?? new List(); + Comparands = comparands ?? new List(); + + OriginalBaselineOtherMetrics = Baselines + .Select(baseline => baseline.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); + + OriginalComparandOtherMetrics = Comparands + .Select(comparand => comparand.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); } - public MicrobenchmarkResult Baseline { get; set; } - public MicrobenchmarkResult Comparand { get; set; } - public List ComparisonResults { get; set; } + public List ComparisonResults { get; set; } + public string BaselineRunName { get; } + public string ComparandRunName { get; } + public string ComparisonName => $"{ComparandRunName} vs {BaselineRunName}"; + public string MicrobenchmarkName { get; } + public IEnumerable Baselines { get; } + public IEnumerable Comparands { get; } - // TODO: Nullable double check. - public string BaselineRunName => Baseline?.Parent?.Name; - public string ComparandRunName => Comparand?.Parent?.Name; - public string MicrobenchmarkName => Baseline.MicrobenchmarkName; + public double[] OutliersFreeBaselineMeanValueCollection => + API.Statistics.RemoveOutliers(Baselines + .Select(baseline => baseline.Statistics?.Mean ?? double.NaN)) + .ToArray(); + public double[] OutliersFreeComparandMeanValueCollection => + API.Statistics.RemoveOutliers(Comparands + .Select(comparand => comparand.Statistics?.Mean ?? double.NaN)) + .ToArray(); - public double MeanDiff => (Comparand.Statistics?.Mean.Value - Baseline.Statistics?.Mean.Value) ?? double.NaN; - public double MeanDiffPerc => (MeanDiff / Baseline.Statistics?.Mean.Value) * 100 ?? double.NaN; + public double AveragedBaselineMeanValue => API.GoodLinq.Average(OutliersFreeBaselineMeanValueCollection, r => r); + public double AveragedComparandMeanValue => API.GoodLinq.Average(OutliersFreeComparandMeanValueCollection, r => r); - public double? GetDiffPercentFromOtherMetrics(string metric) - { - if (!Baseline.OtherMetrics.TryGetValue(metric, out var baselineMetric)) + public double MeanDiff => AveragedComparandMeanValue - AveragedBaselineMeanValue; + public double MeanDiffPerc{ + get { - return null; + if (AveragedBaselineMeanValue == 0) + { + if (AveragedComparandMeanValue == 0) + { + return 0; + } + else + { + return double.NaN; + } + } + return (MeanDiff / AveragedBaselineMeanValue) * 100; } + } - if (!baselineMetric.HasValue) - { - return null; - } + public Dictionary OriginalBaselineOtherMetrics { get; } = new(); + public Dictionary OriginalComparandOtherMetrics { get; } = new(); + public Dictionary OutliersFreeBaselineOtherMetrics => OriginalBaselineOtherMetrics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(x => x.Item1, x => x.Item2); + public Dictionary OutliersFreeComparandOtherMetrics => OriginalComparandOtherMetrics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(x => x.Item1, x => x.Item2); + public Dictionary AveragedBaselineOtherMetrics => OutliersFreeBaselineOtherMetrics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(x => x.Item1, x => x.Item2); + public Dictionary AveragedComparandOtherMetrics => OutliersFreeComparandOtherMetrics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(x => x.Item1, x => x.Item2); - if (!Comparand.OtherMetrics.TryGetValue(metric, out var comparandMetric)) + public Dictionary OtherMetricsDiff => AveragedBaselineOtherMetrics + .Select(kvp => { - return null; - } + if (AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key]); + } + return (kvp.Key, double.NaN); + }) + .ToDictionary(x => x.Item1, x => x.Item2); - if (!comparandMetric.HasValue) + public Dictionary OtherMetricsDiffPerc => AveragedBaselineOtherMetrics + .Select(kvp => { - return null; - } + if (!AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } + if (AveragedBaselineOtherMetrics[kvp.Key] == 0) + { + if (AveragedComparandOtherMetrics[kvp.Key] == 0) + { + return (kvp.Key, 0); + } + else + { + return (kvp.Key, double.NaN); + } + } - return (comparandMetric.Value - baselineMetric.Value) / baselineMetric.Value; - } + if (!OtherMetricsDiff.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } + + if (OtherMetricsDiff[kvp.Key] == double.NaN) + { + return (kvp.Key, double.NaN); + } + + return (kvp.Key, 100 * OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); + }) + .ToDictionary(x => x.Item1, x => x.Item2); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs index 79d2f8eccee..4a20b557394 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs @@ -21,7 +21,7 @@ public MicrobenchmarkComparisonResults(string baselineName, string runName, IEnu public IEnumerable LargeImprovements => Ordered.Where(o => o.MeanDiffPerc < -20).OrderBy(g => g.MeanDiffPerc); public IEnumerable Regressions => Ordered.Where(o => o.MeanDiffPerc < 20 && o.MeanDiffPerc > 5); public IEnumerable Improvements => Ordered.Where(o => o.MeanDiffPerc > -20 && o.MeanDiffPerc < -5).OrderBy(g => g.MeanDiffPerc); - public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)).OrderByDescending(g => g.MeanDiffPerc); + public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)); public IEnumerable StaleImprovements => Ordered.Where((o => o.MeanDiffPerc < 0 && o.MeanDiffPerc > -5)).OrderBy(g => g.MeanDiffPerc); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs new file mode 100644 index 00000000000..f5fb0f898b3 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -0,0 +1,88 @@ +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; +using API = GC.Analysis.API; + +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks +{ + public sealed class MicrobenchmarkResult + { + public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "number of iterations", (Statistics stats) => stats.N }, + { "min", (Statistics stats) => stats.Min }, + { "max", (Statistics stats) => stats.Max }, + { "median", (Statistics stats) => stats.Median }, + { "q1", (Statistics stats) => stats.Q1 }, + { "q3", (Statistics stats) => stats.Q3 }, + { "variance", (Statistics stats) => stats.Variance }, + { "standard deviation", (Statistics stats) => stats.StandardDeviation }, + { "skewness", (Statistics stats) => stats.Skewness }, + { "kurtosis", (Statistics stats) => stats.Kurtosis }, + { "standard error", (Statistics stats) => stats.StandardError }, + { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, + }; + + public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "gc count", (gc) => gc.Stats.Count }, + { "non induced gc count", (gc) => gc.Stats.Count - gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "induced gc count", (gc) => gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "total allocated (mb)", (gc) => gc.Stats.TotalAllocatedMB }, + { "max size peak (mb)", (gc) => gc.Stats.MaxSizePeakMB }, + { "total pause time (msec)", (gc) => gc.Stats.TotalPauseTimeMSec }, + { "gc pause time %", (gc) => gc.Stats.GetGCPauseTimePercentage() }, + { "avg. heap size (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, + { "avg. heap size after (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, + }; + + public MicrobenchmarkResult(string benchmarkFullName, + Run parent, + Benchmark benchmark, + API.GCProcessData? gcData = null, + GCTraceMetrics? gcTraceMetrics = null, + API.CPUProcessData? cpuData = null, + IEnumerable? additionalReportMetrics = null, + IEnumerable? columns = null, + IEnumerable? cpuColumns = null) + { + MicrobenchmarkName = benchmarkFullName; + Parent = parent; + Statistics = benchmark.Statistics; + GCTraceMetrics = gcTraceMetrics; + CPUData = cpuData; + + if (additionalReportMetrics != null) + { + OtherMetrics = benchmark.Metrics + .Where(metric => additionalReportMetrics.Contains(metric.Descriptor.Id)) + .ToDictionary(metric => metric.Descriptor.Id, metric => metric.Value); + } + + if (columns != null) + { + var customStatistics = columns + .Where(column => CustomStatisticsCalculationMap.Keys.Contains(column)) + .Select(column => (column, CustomStatisticsCalculationMap[column](benchmark.Statistics))) + .ToDictionary(x => x.column, x => x.Item2); + + OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(x => x.Key, x => x.Value); + + if (gcData != null) + { + var customGCData = columns + .Where(column => CustomAggregateCalculationMap.Keys.Contains(column)) + .Select(column => (column, CustomAggregateCalculationMap[column](gcData))) + .ToDictionary(x => x.column, x => x.Item2); + + OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(x => x.Key, x => x.Value); + } + } + } + public string MicrobenchmarkName { get; set; } + public Run Parent { get; set; } + public Statistics Statistics { get; set; } + public GCTraceMetrics? GCTraceMetrics { get; set; } + public Dictionary OtherMetrics { get; set; } = new(); + public API.CPUProcessData? CPUData { get; set; } + } +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs new file mode 100644 index 00000000000..9e82ebae66e --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -0,0 +1,346 @@ +using GC.Analysis.API; +using GC.Infrastructure.Core.Configurations; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Newtonsoft.Json; +using System.Collections.Concurrent; +using System.Text.RegularExpressions; + +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks +{ + public static class MicrobenchmarkResultComparison + { + private static readonly int _CPUCount = System.Environment.ProcessorCount; + private static readonly Dictionary _benchmarkNameToTraceFilePatternMap = new() + { + { "ByteMark.BenchBitOps", "ByteMark.BenchBitOps"}, + { "System.Collections.CtorGivenSize.Array(Size: 512)", "System.Collections.CtorGivenSize_String_.Array_size_512_"}, + { "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor(Size: 512)", "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor_size_512_"}, + { "System.IO.Tests.Perf_File.ReadAllBytes(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytes_size_104857600_"}, + { "System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytesAsync_size_104857600_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: ICollection)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: IEnumerable)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Numerics.Tests.Perf_BigInteger.Add(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Add_arguments_65536_"}, + { "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Subtract_arguments_65536_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_10000_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, + }; + + public static ConcurrentBag> LoadBdnJsonResults(MicrobenchmarkConfiguration configuration) + { + ConcurrentBag> bdnJsonResults = new(); + Parallel.ForEach(configuration.Runs, (run) => + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); + string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); + Parallel.ForEach(jsonFiles, jsonPath => + { + run.Value.Name ??= run.Key; + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + if (results != null) + { + bdnJsonResults.Add(new(run.Value, results, jsonPath)); + } + }); + }); + return bdnJsonResults; + } + + // TODO: We should specify relationship between json files and trace files before running benchmarks instead of relying on file name patterns. + // This will make the mapping more robust and less prone to errors due to file naming. + public static Dictionary MapJsonToTrace(string outputPath, ConcurrentBag> bdnJsonResults) + { + Dictionary jsonToTrace = new(); + foreach (var groupForRun in bdnJsonResults.GroupBy(t => t.Item1)) + { + var run = groupForRun.Key; + + // GroupBy(t => t.Item2.Benchmarks.First().FullName) is not a bug: + // In single *full.json, multiple benchmarks stands for multiple input parameter combinations for the same benchmark + // If trace collection is enabled, process data for all those parameter combinations will be in the same trace file + foreach (var g in groupForRun.GroupBy(t => t.Item2.Benchmarks.First().FullName)) + { + var benchmarkName = g.Key; + var sortedJsonFiles = GoodLinq.Select(g, t => t.Item3) + .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) + .ToArray(); + + if (!_benchmarkNameToTraceFilePatternMap.ContainsKey(benchmarkName)) + { + throw new InvalidOperationException($"Benchmark name {benchmarkName} does not have a corresponding trace file pattern in the map."); + } + var traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkName]; + string outputPathForRun = Path.Combine(outputPath, run.Name!); + var sortedTraceFiles = Directory.GetFiles(outputPathForRun, $"{traceFileNameTemplate}*.etl.zip", SearchOption.TopDirectoryOnly) + .OrderBy(traceFile => + { + var match = Regex.Match(Path.GetFileName(traceFile), @"_(\d+)\.etl\.zip$"); + return match.Success ? int.Parse(match.Groups[1].Value) : 0; + }) + .ToArray(); + + if (sortedJsonFiles.Length != sortedTraceFiles.Length) + { + throw new InvalidOperationException( + $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkName}"); + } + + for (int i = 0; i < sortedJsonFiles.Length; i++) + { + jsonToTrace[sortedJsonFiles[i]] = sortedTraceFiles[i]; + } + } + } + + return jsonToTrace; + } + + public static ConcurrentBag + AnalyzeMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, + ConcurrentBag> bdnJsonResults, + bool excludeTraces = false) + { + ConcurrentBag microbenchmarkResults = new(); + + Dictionary jsonToTraceMap = new(); + if ((!excludeTraces) && (configuration.TraceConfigurations?.Type ?? "none") != "none") + { + jsonToTraceMap = MapJsonToTrace(configuration.Output.Path, bdnJsonResults); + } + + ParallelOptions options = new() + { + MaxDegreeOfParallelism = _CPUCount * 2 + }; + + int count = 0; + object _lock = new(); + + Parallel.ForEach(bdnJsonResults, options, t => + { + var run = t.Item1; + var bdnJsonResult = t.Item2; + var jsonPath = t.Item3; + + List? benchmarks = bdnJsonResult?.Benchmarks; + + if (benchmarks == null) + { + return; + } + + if ((!excludeTraces) && configuration.TraceConfigurations?.Type != "none") + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name!); + + if (!jsonToTraceMap.TryGetValue(jsonPath, out string? tracePath) || string.IsNullOrWhiteSpace(tracePath)) + { + throw new InvalidOperationException($"Trace collection is enabled, but no trace path mapping was found for benchmark result '{jsonPath}'."); + } + using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) + { + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); + + foreach (var benchmark in benchmarks) + { + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + + MicrobenchmarkResult? microbenchmarkResult = null; + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) + { + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + { + benchmarkGCData = process; + break; + } + } + if (benchmarkGCData != null) + { + int processID = benchmarkGCData.ProcessID; + + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + { + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); + } + */ + + if (benchmarkGCData.GCs.Count > 0) + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + + microbenchmarkResults.Add(microbenchmarkResult!); + } + } + } + else + { + foreach (var benchmark in benchmarks) + { + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + + MicrobenchmarkResult? microbenchmarkResult = null; + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + microbenchmarkResults.Add(microbenchmarkResult!); + } + } + + lock (_lock) + { + count = count + 1; + Console.Write($"\r{count}/{bdnJsonResults.Count} BDN results analyzed."); + } + }); + + Console.WriteLine(); + return microbenchmarkResults; + } + + public static List CompareMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, IEnumerable microbenchmarkResults, bool excludeTraces = false) + { + bool includeTraces = (!excludeTraces) && (configuration.TraceConfigurations.Type != "none"); + var microbenchmarkResultsGroupedByBenchmarkName = microbenchmarkResults + .GroupBy(microbenchmarkResult => microbenchmarkResult.MicrobenchmarkName); + + List comparisonResults = new(); + object _lock = new(); + ParallelOptions options = new() + { + MaxDegreeOfParallelism = _CPUCount + }; + Parallel.ForEach(microbenchmarkResultsGroupedByBenchmarkName, options, microbenchmarkResultsGroup => + { + if (configuration.Output.run_comparisons != null) + { + foreach (var comparison in configuration.Output.run_comparisons) + { + string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); + string baselineName = breakup[0]; + string runName = breakup[1]; + + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == baselineName); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == runName); + + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults ==null) + { + continue; + } + + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + continue; + } + + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } + } + + // Default case where the run comparisons aren't specified. + else + { + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.is_baseline); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => !r.Parent.is_baseline); + + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults == null) + { + return; + } + + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + return; + } + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } + }); + + return comparisonResults; + } + + public static List GroupComparisonResultsByName(MicrobenchmarkConfiguration configuration, List comparisonResultForAllBenchmarks, bool excludeTraces = false) + { + List allComparisonResults = new(); + + comparisonResultForAllBenchmarks + .GroupBy(r => r.ComparisonName) + .ToList() + .ForEach(group => + { + string baselineName = group.FirstOrDefault()?.BaselineRunName ?? "Baseline"; + string runName = group.FirstOrDefault()?.ComparandRunName ?? "Comparand"; + allComparisonResults.Add(new(baselineName, runName, group.ToList())); + }); + + return allComparisonResults; + } + } +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs deleted file mode 100644 index 4ad9158e74c..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs +++ /dev/null @@ -1,178 +0,0 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using Newtonsoft.Json; -using System.Collections.Concurrent; - -namespace GC.Infrastructure.Core.Analysis.Microbenchmarks -{ - public static class MicrobenchmarkResultsAnalyzer - { - public static IReadOnlyDictionary> Analyze(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - ConcurrentDictionary> runsToResults = new(); - - Parallel.ForEach(configuration.Runs, (run) => - { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); - run.Value.Name ??= run.Key; - - // Find the json path. - string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); - - // Retrieve benchmarks from all the JSON files. - Parallel.ForEach(jsonFiles, (jsonFile) => - { - MicrobenchmarkResults results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - foreach (var benchmark in results?.Benchmarks) - { - string title = benchmark.FullName; - Statistics statistics = benchmark.Statistics; - - if (!runsToResults.TryGetValue(run.Value, out var perBenchmarkData)) - { - runsToResults[run.Value] = perBenchmarkData = new ConcurrentDictionary(); - } - - runsToResults[run.Value][title] = new MicrobenchmarkResult - { - Statistics = statistics, - Parent = run.Value, - MicrobenchmarkName = title, - }; - } - }); - - if (!excludeTraces) - { - Dictionary analyzers = AnalyzerManager.GetAllAnalyzers(outputPathForRun); - - foreach (var analyzer in analyzers) - { - List allPertinentProcesses = analyzer.Value.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.Value.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); - foreach (var benchmark in runsToResults[run.Value]) - { - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) - { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.Key.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) - { - benchmarkGCData = process; - break; - } - } - - if (benchmarkGCData != null) - { - int processID = benchmarkGCData.ProcessID; - benchmark.Value.GCData = benchmarkGCData; - benchmark.Value.ResultItem = new Presentation.GCPerfSim.ResultItem(benchmarkGCData, analyzer.Key, benchmark.Key); - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) - { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); - } - */ - } - } - }; - } - }); - - return runsToResults; - } - - public static IReadOnlyList GetComparisons(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - IReadOnlyDictionary> runResults = Analyze(configuration, excludeTraces); - List comparisonResults = new(); - - if (configuration.Output.run_comparisons != null) - { - foreach (var comparison in configuration.Output.run_comparisons) - { - string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); - string baselineName = breakup[0]; - string runName = breakup[1]; - - Run run = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, runName) == 0); - Run baselineRun = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, baselineName) == 0); - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResults[run]) - { - string microbenchmarkName = r.Key; - if (runResults[baselineRun].TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - } - } - - // At this point of time, the lack thereof of either of the runs should be a non-issue. - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - // Default case where the run comparisons aren't specified. - else - { - string baselineName = configuration.Runs.FirstOrDefault(r => r.Value.is_baseline).Key; - KeyValuePair> baselineResult = baselineName != null ? runResults.First(r => r.Key.Name == baselineName) : runResults.First(); - - // For each run, we want to grab it and it's baseline and then do a per microbenchmark association. - foreach (var runResult in runResults) - { - Run run = runResult.Key; - string runName = run.Name; - - if (string.CompareOrdinal(runName, baselineName) == 0) - { - continue; - } - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResult.Value) - { - string microbenchmarkName = r.Key; - if (baselineResult.Value.TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - } - } - - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - return comparisonResults; - } - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs index fb2546f9bb7..b0ca7613bc6 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs @@ -13,6 +13,8 @@ public sealed class InputConfiguration // public Dictionary? clrgcs { get; set; } // public string? debug_parameters { get; set; } + public Dictionary? iterations { get; set; } + public Dictionary? symbol_path { get; set; } public Dictionary? source_path { get; set; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs index c44e02cd947..ab9ab01a911 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs @@ -24,7 +24,13 @@ public sealed class Run : RunBase public class Environment { public uint default_max_seconds { get; set; } = 300; - public uint iteration { get; set; } = 1; + public uint iterations { get; set; } = 1; + + [YamlMember(Alias = "iteration")] + public uint iteration + { + set => iterations = value; + } } public sealed class MicrobenchmarkConfigurations diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs similarity index 50% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs index 3d547c03a0c..9bf2736ef0a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs @@ -1,14 +1,16 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.GCPerfSim; using Newtonsoft.Json; -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json +namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Json { - public static void Generate(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, string path) + public static void Generate(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, string path) { - string json = JsonConvert.SerializeObject(comparisonResults); + string json = JsonConvert.SerializeObject(comparisonResultsGroupedByName); File.WriteAllText(path, json); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs deleted file mode 100644 index 9840432b2ba..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json -{ - public sealed class JsonOutput - { - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 76dffd7f87d..dca122b017a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -1,33 +1,30 @@ -using API = GC.Analysis.API; -using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Analysis.API; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using System.Configuration; +using API = GC.Analysis.API; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Markdown { - private const string baseTableString = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; - private const string baseTableRows = "| --- | --- | -- | --- | --- | --- | --- | "; - - public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, Dictionary executionDetails, string path) + public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResultsCollection, Dictionary executionDetails, string path) { using (StreamWriter sw = new StreamWriter(path)) { // Create summary. sw.WriteLine("# Summary"); - string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResults, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; + string header = $"| Criteria | {string.Join("|", API.GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; sw.WriteLine(header); - sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResults.Count))} |"); - sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeRegressions.Count())}|"); - sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Regressions.Count())}|"); - sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleRegressions.Count())}|"); - sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleImprovements.Count())}|"); - sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Improvements.Count())}|"); - sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeImprovements.Count())}|"); - sw.WriteLine($"| Total | {comparisonResults.Count} |"); + sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResultsCollection.Count))} |"); + sw.WriteLine($"| Large Regressions (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); + sw.WriteLine($"| Regressions (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); + sw.WriteLine($"| Stale Regressions (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); + sw.WriteLine($"| Stale Improvements (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); + sw.WriteLine($"| Improvements (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); + sw.WriteLine($"| Large Improvements (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); + sw.WriteLine($"| Total | {comparisonResultsCollection.Count} |"); sw.WriteLine("\n"); // Incomplete Tests. @@ -41,9 +38,9 @@ public static void GenerateTable(MicrobenchmarkConfiguration configuration, IRea sw.WriteLine("## Individual Results"); // Add details of Each Comparison. - foreach (var comparisonResult in comparisonResults) + foreach (var comparisonResults in comparisonResultsCollection) { - AddDetailsOfSingleComparison(sw, configuration, comparisonResult); + AddDetailsOfSingleComparison(sw, configuration, comparisonResults); sw.WriteLine("\n"); } } @@ -89,52 +86,65 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben foreach (var metric in configuration.Output.additional_report_metrics) { sw.WriteLine($"## Comparison by {metric}"); - var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.GetDiffPercentFromOtherMetrics(metric)); + var ordered = comparisonResult.Comparisons + .Where(c => c.OtherMetricsDiffPerc.ContainsKey(metric)) + .OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); // Large Regressions - sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.2)); + var largeRegression = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 20); + sw.WriteLine($"### Large Regressions (>20%): {largeRegression.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeRegression, metric); sw.WriteLine("\n"); // Large Improvements - sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - var largeImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < -0.2); + var largeImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -20); largeImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, largeImprovements); + sw.WriteLine($"### Large Improvements (>20%): {largeImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeImprovements, metric); sw.WriteLine("\n"); // Regressions - sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2)); + var regressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 5 && o.OtherMetricsDiffPerc[metric] < 20); + sw.WriteLine($"### Regressions (5% - 20%): {regressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, regressions, metric); sw.WriteLine("\n"); // Improvements - sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - var improvements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2); + var improvements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -5 && o.OtherMetricsDiffPerc[metric] > -20); improvements.Reverse(); - sw.AddTableForSingleCriteria(configuration, improvements); + sw.WriteLine($"### Improvements (5% - 20%): {improvements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, improvements, metric); sw.WriteLine("\n"); // Stale Regressions - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < 0.05 && o.GetDiffPercentFromOtherMetrics(metric) >= 0.0)); + var staleRegressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 0.0 && o.OtherMetricsDiffPerc[metric] < 5); + sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {staleRegressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleRegressions, metric); sw.WriteLine("\n"); // Stale Improvements - sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - var staleImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > -0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.0); + var staleImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -5 && o.OtherMetricsDiffPerc[metric] <= 0.0); staleImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, staleImprovements); + sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {staleImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleImprovements, metric); sw.WriteLine("\n"); } } } - internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons) + internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons, string? metricName = null) { // Check if all comparisons have traces. - string tableHeader0 = baseTableString; - string tableHeader1 = baseTableRows; + string tableHeader0 = ""; + if (!string.IsNullOrEmpty(metricName)) + { + tableHeader0 = $"| Benchmark Name | Baseline | Comparand | Baseline {metricName} | Comparand {metricName} | Δ {metricName} | Δ% {metricName} |"; + } + else + { + tableHeader0 = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; + } + string tableHeader1 = "| --- | --- | -- | --- | --- | --- | --- | "; if (configuration.Output.Columns != null) { @@ -145,14 +155,15 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; - tableHeader1 += "--- | --- | --- | --- |"; - } - } + // TODO: Add CPU columns if needed in the future. + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; + // tableHeader1 += "--- | --- | --- | --- |"; + // } + //} sw.WriteLine(tableHeader0); sw.WriteLine(tableHeader1); @@ -161,24 +172,27 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm { try { - var baseRow = $"| {lr.MicrobenchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.Baseline.Statistics.Mean.Value, 2)} | {Math.Round(lr.Comparand.Statistics.Mean.Value, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + string benchmarkName = lr.MicrobenchmarkName.Replace("<", "\\<").Replace(">", "\\>"); + string baseRow = ""; + + if (!String.IsNullOrEmpty(metricName)) + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineOtherMetrics[metricName], 2)} | {Math.Round(lr.AveragedComparandOtherMetrics[metricName], 2)} | {Math.Round(lr.OtherMetricsDiff[metricName], 2)}| {Math.Round(lr.OtherMetricsDiffPerc[metricName], 2)}|"; + } + else + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + } if (configuration.Output.Columns != null) { foreach (var column in configuration.Output.Columns) { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) - { - lr.Baseline.OtherMetrics[column] = baselineValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Baseline.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Baseline); - } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; + double? baselineValue = lr.AveragedBaselineOtherMetrics.GetValueOrDefault(column); + double? comparandValue = lr.AveragedComparandOtherMetrics.GetValueOrDefault(column); - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) - { - lr.Comparand.OtherMetrics[column] = comparandValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Comparand.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Comparand); - } + string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 4).ToString() : string.Empty; - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; string deltaResult = delta.HasValue ? Math.Round(delta.Value, 4).ToString() : string.Empty; @@ -189,31 +203,32 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) - { - lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; - - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) - { - lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; - - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; - string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; - - double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; - string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; - - baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; - } - } + // TODO: Add CPU columns if needed in the future. + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) + // { + // lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; + + // if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) + // { + // lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; + + // double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; + // string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; + + // double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; + // string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; + + // baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; + // } + //} sw.WriteLine(baseRow); } @@ -224,15 +239,6 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm Console.WriteLine(e.StackTrace); } } - - // Dispose all the Analyzers now that we are persisting the values. - foreach (var comparison in comparisons) - { - comparison.Baseline?.GCData?.Parent?.Dispose(); - comparison.Baseline?.CPUData?.Parent?.Analyzer?.Dispose(); - comparison.Comparand?.GCData?.Parent?.Dispose(); - comparison.Comparand?.CPUData?.Parent?.Analyzer?.Dispose(); - } } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs deleted file mode 100644 index 93f4faea7f0..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ /dev/null @@ -1,31 +0,0 @@ -using GC.Infrastructure.Core.Analysis; -using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Configurations; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks -{ - public static class Presentation - { - public static IReadOnlyList Present(MicrobenchmarkConfiguration configuration, Dictionary executionDetails) - { - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); - foreach (var format in configuration.Output.Formats) - { - if (format == "markdown") - { - Markdown.GenerateTable(configuration, comparisonResults, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); - continue; - } - - if (format == "json") - { - Json.Json.Generate(configuration, comparisonResults, Path.Combine(configuration.Output.Path, "Results.json")); - continue; - } - } - - return comparisonResults; - } - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 35c4e50a266..b179894d524 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,10 +1,12 @@ -using Spectre.Console.Cli; -using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; +using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using Spectre.Console; +using Spectre.Console.Cli; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -21,9 +23,46 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben { ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); - Presentation.Present(configuration, new()); // Execution details aren't available for the analysis-only mode. + + var comparisonResultsGroupedByName = ExecuteAnalysis(configuration); + + Present(configuration, comparisonResultsGroupedByName, new()); // Execution details aren't available for the analysis-only mode. return 0; } + + public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) + { + var bdnJsonResults = MicrobenchmarkResultComparison.LoadBdnJsonResults(configuration); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) {bdnJsonResults.Count} BDN results loaded.[/]"); + var microbenchmarkResults = MicrobenchmarkResultComparison.AnalyzeMicrobenchmarkResults(configuration, bdnJsonResults); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Analysis completed.[/]"); + var comparisonResults = MicrobenchmarkResultComparison.CompareMicrobenchmarkResults(configuration, microbenchmarkResults); + + return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResults); + } + + public static void Present(MicrobenchmarkConfiguration configuration, + List comparisonResultsGroupedByName, + Dictionary executionDetails) + { + foreach (var format in configuration.Output.Formats) + { + if (format == "markdown") + { + string outputPath = Path.Combine(configuration.Output.Path, "Results.md"); + Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); + continue; + } + + if (format == "json") + { + string outputPath = Path.Combine(configuration.Output.Path, "Results.json"); + Json.Generate(configuration, comparisonResultsGroupedByName, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); + continue; + } + } + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index ca063389ef5..e411bbd2f65 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -1,18 +1,17 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.CommandBuilders; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Configurations; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.TraceCollection; using Newtonsoft.Json; -using Spectre.Console.Cli; using Spectre.Console; +using Spectre.Console.Cli; using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Text; -using System.Configuration; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -121,11 +120,11 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi // Should only be one if it's a fresh run. string jsonFile = jsonFiles.First(); - MicrobenchmarkResults output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + BdnJsonResult output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); // Assumption: A particular run, regardless of the parameters, will run ~the same vals. - IEnumerable operationsPerNanos = output.Benchmarks.First().Measurements.Where(m => m.IterationMode == "Workload" && m.IterationStage == "Actual") - .Select(m => m.Operations); + var operationsPerNanos = GoodLinq.Select(GoodLinq.Where(output.Benchmarks.First().Measurements, m => m.IterationMode == "Workload" && m.IterationStage == "Actual"), m => m.Operations); + // For now take the max but we will possibly be sacrificing duration for precision. invocationCountFromBaseline = operationsPerNanos.Max(); } @@ -149,9 +148,9 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi (string, string) fileNameAndCommand = MicrobenchmarkCommandBuilder.Build(configuration, run, benchmark, invocationCountFromBaseline); run.Value.Name = run.Key; - for (int index = 0; index < configuration.Environment.iteration; index++) + for (int index = 0; index < configuration.Environment.iterations; index++) { - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbechmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbenchmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); // Run The BDN process with the trace collector. using (Process bdnProcess = new()) { @@ -199,10 +198,12 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi } } - IReadOnlyList results = Presentation.Present(configuration, executionDetails); + var comparisonResultsGroupedName = MicrobenchmarkAnalyzeCommand.ExecuteAnalysis(configuration); + + MicrobenchmarkAnalyzeCommand.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); - return new MicrobenchmarkOutputResults(executionDetails, results); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbenchmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); + return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml index ea719bad713..257f021c024 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml @@ -6,7 +6,7 @@ microbenchmark_configurations: environment: default_max_seconds: 3000 - iteration: 1 + iterations: 1 # Configurations that involve capturing a trace. trace_configurations: diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt index 1bef991eca5..6056a312aff 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt @@ -11,7 +11,6 @@ "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 1000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | -"System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)" | "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536*" | "System.Collections.CtorGivenSize.Array(size: 512)" | "ByteMark.BenchBitOps" | diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs index 0cec453d0c3..96a6943019b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs @@ -225,6 +225,12 @@ internal static MicrobenchmarkConfiguration CreateBaseMicrobenchmarkSuite(InputC }); } + // Set iterations if they exist. + if (inputConfiguration.iterations != null) + { + configuration.Environment.iterations = inputConfiguration.iterations.GetValueOrDefault("microbenchmarks", 1); + } + // The first run is always the baseline. configuration.Runs.First().Value.is_baseline = true;