Average multi microbenchmarks results

src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml

@@ -12,6 +12,10 @@ coreruns:
       environment_variables:
           DOTNET_GCName: clrgc.dll
 
+iterations:
+  gcperfsim: 1
+  microbenchmarks: 1
+
 trace_configuration_type: gc # Choose between: none, gc, verbose, cpu, cpu_managed, threadtime, join.
 
 # Optional fields: the contents of both the symbol_path and the source_path will be copied over to the output path.

src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs

@@ -31,5 +31,26 @@ public static double StandardDeviation(this IEnumerable<double> doubleList)
             double sumOfDerivationAverage = sumOfDerivation / (doubleList.Count() - 1);
             return Math.Sqrt(sumOfDerivationAverage - (average * average));
         }
+
+        public static IEnumerable<double> RemoveOutliers(IEnumerable<double> collection)
+        {
+            if (!collection.Any())
+            {
+                return Array.Empty<double>();
+            }
+            // Calculate Q1 (25th percentile) and Q3 (75th percentile)
+            double q1 = GC.Analysis.API.Statistics.Percentile(collection, 0.25);
+            double q3 = GC.Analysis.API.Statistics.Percentile(collection, 0.75);
+
+            // Calculate IQR (Interquartile Range)
+            double iqr = q3 - q1;
+
+            // Calculate bounds: [Q1 - 1.5*IQR, Q3 + 1.5*IQR]
+            double lowerBound = q1 - 1.5 * iqr;
+            double upperBound = q3 + 1.5 * iqr;
+
+            // Filter out outliers
+            return GoodLinq.Where(collection, x => x >= lowerBound && x <= upperBound);
+        }
     }
 }

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs → src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs

@@ -1,103 +1,10 @@
-using GC.Analysis.API;
-using GC.Infrastructure.Core.Configurations.Microbenchmarks;
-using GC.Infrastructure.Core.Presentation.GCPerfSim;
-using Newtonsoft.Json;
-
-namespace GC.Infrastructure.Core.Analysis
+namespace GC.Infrastructure.Core.Analysis
 {
-    public sealed class MicrobenchmarkResult
-    {
-        public Statistics Statistics { get; set; }
-
-        [JsonIgnore]
-        public GCProcessData? GCData { get; set; }
-
-        public ResultItem ResultItem { get; set; }
-
-        [JsonIgnore]
-        public CPUProcessData? CPUData { get; set; }
-        public Run Parent { get; set; }
-        public string MicrobenchmarkName { get; set; }
-        public Dictionary<string, double?> OtherMetrics { get; set; } = new();
-
-        private static readonly IReadOnlyDictionary<string, Func<Statistics, double?>> _customStatisticsCalculationMap = new Dictionary<string, Func<Statistics, double?>>(StringComparer.OrdinalIgnoreCase)
-        {
-            { "number of iterations", (Statistics stats) => stats.N },
-            { "min", (Statistics stats) => stats.Min },
-            { "max", (Statistics stats) => stats.Max },
-            { "median", (Statistics stats) => stats.Median },
-            { "q1", (Statistics stats) => stats.Q1 },
-            { "q3", (Statistics stats) => stats.Q3 },
-            { "variance", (Statistics stats) => stats.Variance },
-            { "standard deviation", (Statistics stats) => stats.StandardDeviation },
-            { "skewness", (Statistics stats) => stats.Skewness },
-            { "kurtosis", (Statistics stats) => stats.Kurtosis },
-            { "standard error", (Statistics stats) => stats.StandardError },
-            { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean },
-        };
-
-        public static double? LookupStatisticsCalculation(string columnName, MicrobenchmarkResult result)
-        {
-            if (string.IsNullOrEmpty(columnName))
-            {
-                return null;
-            }
-
-            if (!_customStatisticsCalculationMap.TryGetValue(columnName, out var val))
-            {
-                return null;
-            }
-
-            else
-            {
-                return val.Invoke(result.Statistics);
-            }
-        }
-    }
-
-    public sealed class Benchmark
-    {
-        public string DisplayInfo { get; set; }
-        public string Namespace { get; set; }
-        public string Type { get; set; }
-        public string Method { get; set; }
-        public string MethodTitle { get; set; }
-        public string Parameters { get; set; }
-        public string FullName { get; set; }
-        public Statistics Statistics { get; set; }
-        public Memory Memory { get; set; }
-        public List<Measurement> Measurements { get; set; }
-        public List<Metric> Metrics { get; set; }
-    }
-
     public sealed class ChronometerFrequency
     {
         public int Hertz { get; set; }
     }
 
-    public sealed class ConfidenceInterval
-    {
-        public int N { get; set; }
-        public double? Mean { get; set; }
-        public double? StandardError { get; set; }
-        public int? Level { get; set; }
-        public double? Margin { get; set; }
-        public double? Lower { get; set; }
-        public double? Upper { get; set; }
-    }
-
-    public sealed class Descriptor
-    {
-        public string Id { get; set; }
-        public string DisplayName { get; set; }
-        public string Legend { get; set; }
-        public string NumberFormat { get; set; }
-        public int UnitType { get; set; }
-        public string Unit { get; set; }
-        public bool TheGreaterTheBetter { get; set; }
-        public int PriorityInCategory { get; set; }
-    }
-
     public sealed class HostEnvironmentInfo
     {
         public string BenchmarkDotNetCaption { get; set; }
@@ -117,6 +24,16 @@ public sealed class HostEnvironmentInfo
         public string HardwareTimerKind { get; set; }
     }
 
+
+    public sealed class Memory
+    {
+        public int Gen0Collections { get; set; }
+        public int Gen1Collections { get; set; }
+        public int Gen2Collections { get; set; }
+        public int TotalOperations { get; set; }
+        public long BytesAllocatedPerOperation { get; set; }
+    }
+
     public sealed class Measurement
     {
         public string IterationMode { get; set; }
@@ -127,19 +44,27 @@ public sealed class Measurement
         public long Nanoseconds { get; set; }
     }
 
-    public sealed class Memory
+    public sealed class Descriptor
     {
-        public int Gen0Collections { get; set; }
-        public int Gen1Collections { get; set; }
-        public int Gen2Collections { get; set; }
-        public int TotalOperations { get; set; }
-        public long BytesAllocatedPerOperation { get; set; }
+        public string Id { get; set; }
+        public string DisplayName { get; set; }
+        public string Legend { get; set; }
+        public string NumberFormat { get; set; }
+        public int UnitType { get; set; }
+        public string Unit { get; set; }
+        public bool TheGreaterTheBetter { get; set; }
+        public int PriorityInCategory { get; set; }
     }
 
-    public sealed class Metric
+    public sealed class ConfidenceInterval
     {
-        public double Value { get; set; }
-        public Descriptor Descriptor { get; set; }
+        public int N { get; set; }
+        public double? Mean { get; set; }
+        public double? StandardError { get; set; }
+        public int? Level { get; set; }
+        public double? Margin { get; set; }
+        public double? Lower { get; set; }
+        public double? Upper { get; set; }
     }
 
     public sealed class Percentiles
@@ -155,13 +80,6 @@ public sealed class Percentiles
         public double P100 { get; set; }
     }
 
-    public sealed class MicrobenchmarkResults
-    {
-        public string Title { get; set; }
-        public HostEnvironmentInfo HostEnvironmentInfo { get; set; }
-        public List<Benchmark> Benchmarks { get; set; }
-    }
-
     public sealed class Statistics
     {
         public List<double> OriginalValues { get; set; }
@@ -186,4 +104,32 @@ public sealed class Statistics
         public ConfidenceInterval? ConfidenceInterval { get; set; }
         public Percentiles Percentiles { get; set; }
     }
-}
+
+    public sealed class Metric
+    {
+        public double Value { get; set; }
+        public Descriptor Descriptor { get; set; }
+    }
+
+    public sealed class Benchmark
+    {
+        public string DisplayInfo { get; set; }
+        public string Namespace { get; set; }
+        public string Type { get; set; }
+        public string Method { get; set; }
+        public string MethodTitle { get; set; }
+        public string Parameters { get; set; }
+        public string FullName { get; set; }
+        public Statistics Statistics { get; set; }
+        public Memory Memory { get; set; }
+        public List<Measurement> Measurements { get; set; }
+        public List<Metric> Metrics { get; set; }
+    }
+
+    public sealed class BdnJsonResult
+    {
+        public string Title { get; set; }
+        public HostEnvironmentInfo HostEnvironmentInfo { get; set; }
+        public List<Benchmark> Benchmarks { get; set; }
+    }
+}

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs

@@ -0,0 +1,8 @@
+namespace GC.Infrastructure.Core.Analysis
+{
+    public static class GCTraceMetricComparison
+    {
+        public static GCTraceMetricComparisonResult CompareGCTraceMetric(IEnumerable<GCTraceMetrics> baselines, IEnumerable<GCTraceMetrics> comparands,string nameOfMetric)
+            => new GCTraceMetricComparisonResult(baselines, comparands, nameOfMetric);
+    }
+}

src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs

@@ -0,0 +1,100 @@
+using GC.Analysis.API;
+using Microsoft.Diagnostics.Tracing.Analysis.GC;
+using System.Reflection;
+
+namespace GC.Infrastructure.Core.Analysis
+{
+    public sealed class GCTraceMetricComparisonResult
+    {
+        public GCTraceMetricComparisonResult(IEnumerable<GCTraceMetrics> baselines, IEnumerable<GCTraceMetrics> comparands, string metricName)
+        {
+            RunName = baselines.FirstOrDefault()?.RunName;
+            Key = $"{baselines.FirstOrDefault()?.ConfigurationName}_{RunName}";
+
+            MetricName = metricName;
+            PropertyInfo pInfo = typeof(GCTraceMetrics).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public);
+
+            // Property found on the GCTraceMetrics.
+            if (pInfo != null)
+            {
+                OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)pInfo.GetValue(baseline));
+                OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)pInfo.GetValue(comparand));
+            }
+
+            // If property isn't found on the GCTraceMetrics, look in GCStats.
+            // TODO: Add the case where we look into the map.
+            else
+            {
+                pInfo = typeof(GCStats).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public);
+                if (pInfo == null)
+                {
+                    FieldInfo fieldInfo = typeof(GCStats).GetField(metricName, BindingFlags.Instance | BindingFlags.Public);
+                    if (fieldInfo == null)
+                    {
+                        // Out of luck!
+                        OriginalBaselineMetricCollection = Array.Empty<double>();
+                        OriginalComparandMetricCollection = Array.Empty<double>();
+                        OutliersFreeBaselineMetricCollection = Array.Empty<double>();
+                        OutliersFreeComparandMetricCollection = Array.Empty<double>();
+                        AveragedBaselineMetric = double.NaN;
+                        AveragedComparandMetric = double.NaN;
+                        return;
+                    }
+
+                    else
+                    {
+                        OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)fieldInfo.GetValue(baseline));
+                        OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)fieldInfo.GetValue(comparand));
+                    }
+                }
+
+                else
+                {
+                    OriginalBaselineMetricCollection = GoodLinq.Select(baselines, baseline => (double)pInfo.GetValue(baseline));
+                    OriginalComparandMetricCollection = GoodLinq.Select(comparands, comparand => (double)pInfo.GetValue(comparand));
+                }
+            }
+
+            // Filter out outliers using IQR method
+            OutliersFreeBaselineMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalBaselineMetricCollection);
+            OutliersFreeComparandMetricCollection = GC.Analysis.API.Statistics.RemoveOutliers(OriginalComparandMetricCollection);
+
+            // Calculate averaged metrics
+            AveragedBaselineMetric = GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r);
+            AveragedComparandMetric = GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r);
+        }
+
+        public string RunName { get; }
+        public string Key { get; }
+        public string MetricName { get; }
+        public IEnumerable<double> OriginalBaselineMetricCollection { get; }
+        public IEnumerable<double> OriginalComparandMetricCollection { get; }
+        public IEnumerable<double> OutliersFreeBaselineMetricCollection { get; }
+        public IEnumerable<double> OutliersFreeComparandMetricCollection { get; }
+
+        public double AveragedBaselineMetric { get; }
+        public double AveragedComparandMetric { get; }
+        public double Delta => AveragedComparandMetric - AveragedBaselineMetric;
+        public double PercentageDelta
+        {
+            get
+            {
+                if (AveragedBaselineMetric == 0)
+                {
+                    if (AveragedComparandMetric == 0)
+                    {
+                        return 0;
+                    }
+                    else
+                    {
+                        return double.NaN;
+                    }
+                }
+                else
+                {
+                    return Delta / AveragedBaselineMetric * 100.0;
+                }
+            }
+        }
+    }
+}