Support benchmarks with params (#18)

Benchly.Benchmarks/Md5VsSha256Params.cs

@@ -0,0 +1,35 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Jobs;
+using System.Security.Cryptography;
+
+namespace Benchly.Benchmarks
+{
+    [BoxPlot(Title = "Box Plot")]
+    [BarPlot(Title = "Bar Plot")]
+    [Histogram]
+    [Timeline]
+    [MemoryDiagnoser, SimpleJob(RuntimeMoniker.Net60), SimpleJob(RuntimeMoniker.Net48)]
+    public class Md5VsSha256Params
+    {
+        private byte[] data;
+
+        private readonly SHA256 sha256 = SHA256.Create();
+        private readonly MD5 md5 = MD5.Create();
+
+        [Params(128, 1024, 16384)]
+        public int Size { get; set; }
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            data = new byte[Size];
+            new Random(42).NextBytes(data);
+        }
+
+        [Benchmark]
+        public byte[] Sha256() => sha256.ComputeHash(data);
+
+        [Benchmark]
+        public byte[] Md5() => md5.ComputeHash(data);
+    }
+}

Benchly.UnitTests/TestBenchmarkRunner.cs

@@ -5,14 +5,9 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
-using System;
-using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Globalization;
-using System.Linq;
 using System.Security.Cryptography;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Benchly.UnitTests
 {
@@ -25,37 +20,62 @@ public class TestBenchmarkRunner
                  BenchmarkDotNet.Environments.HostEnvironmentInfo.GetCurrent(),
                  string.Empty, string.Empty, TimeSpan.Zero, CultureInfo.CurrentCulture,
                  new List<ValidationError>().ToImmutableArray(),
-                new List<IColumnHidingRule>().ToImmutableArray());
+                 new List<IColumnHidingRule>().ToImmutableArray());
 
         /// <summary>
         /// Runs a test benchmark.
         /// </summary>
         /// <returns>The summary result of the benchmark.</returns>
-        public Summary GetSummary()
+        public Summary GetSummary<TBenchmark>()
         {
-            return BenchmarkRunner.Run(typeof(Md5VsSha256), DefaultConfig.Instance.WithOptions(ConfigOptions.DisableOptimizationsValidator));
+            return BenchmarkRunner.Run(typeof(TBenchmark), DefaultConfig.Instance.WithOptions(ConfigOptions.DisableOptimizationsValidator));
         }
+    }
+
+    [MemoryDiagnoser, SimpleJob(RuntimeMoniker.Net60), SimpleJob(RuntimeMoniker.Net48)]
+    public class Md5VsSha256
+    {
+        private const int N = 10000;
+        private readonly byte[] data;
 
-        [MemoryDiagnoser, SimpleJob(RuntimeMoniker.Net60), SimpleJob(RuntimeMoniker.Net48)]
-        public class Md5VsSha256
+        private readonly SHA256 sha256 = SHA256.Create();
+        private readonly MD5 md5 = MD5.Create();
+
+        public Md5VsSha256()
         {
-            private const int N = 10000;
-            private readonly byte[] data;
+            data = new byte[N];
+            new Random(42).NextBytes(data);
+        }
+
+        [Benchmark]
+        public byte[] Sha256() => sha256.ComputeHash(data);
 
-            private readonly SHA256 sha256 = SHA256.Create();
-            private readonly MD5 md5 = MD5.Create();
+        [Benchmark]
+        public byte[] Md5() => md5.ComputeHash(data);
+    }
+
+    [MemoryDiagnoser, SimpleJob(RuntimeMoniker.Net60), SimpleJob(RuntimeMoniker.Net48)]
+    public class Md5VsSha256Params
+    {
+        private byte[] data;
 
-            public Md5VsSha256()
-            {
-                data = new byte[N];
-                new Random(42).NextBytes(data);
-            }
+        private readonly SHA256 sha256 = SHA256.Create();
+        private readonly MD5 md5 = MD5.Create();
 
-            [Benchmark]
-            public byte[] Sha256() => sha256.ComputeHash(data);
+        [Params(128, 1024, 16384)]
+        public int Size { get; set; }
 
-            [Benchmark]
-            public byte[] Md5() => md5.ComputeHash(data);
+        [GlobalSetup]
+        public void Setup()
+        {
+            data = new byte[Size];
+            new Random(42).NextBytes(data);
         }
+
+        [Benchmark]
+        public byte[] Sha256() => sha256.ComputeHash(data);
+
+        [Benchmark]
+        public byte[] Md5() => md5.ComputeHash(data);
     }
 }

Benchly.UnitTests/UnitTest1.cs

@@ -1,14 +1,28 @@
 using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
 
 namespace Benchly.UnitTests
 {
     public class UnitTest1
     {
         [Fact]
-        public void Test1()
+        public void TestNoParams()
         {
             var bench = new TestBenchmarkRunner();
-            var summary = bench.GetSummary();
+            var summary = bench.GetSummary<Md5VsSha256>();
+            RunExporters(summary);
+        }
+
+        [Fact]
+        public void TestParams()
+        {
+            var bench = new TestBenchmarkRunner();
+            var summary = bench.GetSummary<Md5VsSha256Params>();
+            RunExporters(summary);
+        }
+
+        private static void RunExporters(Summary summary)
+        {
             var boxPlotExporter = new BoxPlotExporter();
             boxPlotExporter.Info.Title = "Box Plot";
             var barPlotExporter = new BarPlotExporter();

Benchly/BarPlotExporter.cs

@@ -3,20 +3,43 @@
 using Plotly.NET.ImageExport;
 using BenchmarkDotNet.Loggers;
 using Plotly.NET;
+using Plotly.NET.LayoutObjects;
+using static Plotly.NET.StyleParam;
+using Microsoft.FSharp.Core;
 
 namespace Benchly
 {
     internal class BarPlotExporter : IExporter
     {
-        public BarPlotExporter()
-        {
-        }
-
         public PlotInfo Info { get; set; } = new PlotInfo();
 
         public string Name => nameof(BarPlotExporter);
 
         public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
+        {
+            if (summary.Reports.Length == 0)
+            { 
+                return Array.Empty<string>();
+            }
+
+            if (summary.Reports[0].BenchmarkCase.HasParameters)
+            {
+                int paramCount = summary.Reports[0].BenchmarkCase.Parameters.Count;
+
+                if (paramCount == 1)
+                {
+                    return OneParameter(summary);
+                }
+            }
+
+            return NoParameter(summary);
+        }
+
+        public void ExportToLog(Summary summary, ILogger logger)
+        {
+        }
+
+        private IEnumerable<string> NoParameter(Summary summary)
         {
             var title = this.Info.Title ?? summary.Title;
             var file = Path.Combine(summary.ResultsDirectoryPath, ExporterBase.GetFileName(summary) + "-barplot");
@@ -25,9 +48,9 @@ public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
 
             var colors = ColorMap.GetJobColors(summary, this.Info);
 
-            var jobs = summary.Reports.Select(r => new 
-            { 
-                job = r.BenchmarkCase.Job.ResolvedId, 
+            var jobs = summary.Reports.Select(r => new
+            {
+                job = r.BenchmarkCase.Job.ResolvedId,
                 name = r.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo,
                 mean = r.Success ? ConvertNanosToMs(r.ResultStatistics.Mean) : 0
             }).GroupBy(r => r.job);
@@ -49,16 +72,81 @@ public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
             return new[] { file + ".svg" };
         }
 
+        private IEnumerable<string> OneParameter(Summary summary)
+        {
+            var title = this.Info.Title ?? summary.Title;
+            var file = Path.Combine(summary.ResultsDirectoryPath, ExporterBase.GetFileName(summary) + "-barplot");
+
+            // make a grid with 1 row, n columns, where n is number of params
+            // y axis only on first chart
+            var gridCharts = new List<GenericChart.GenericChart>();
+
+            var colors = ColorMap.GetJobColors(summary, this.Info);
+
+            var byParam = summary.Reports.Select(r => new
+            {
+                param = r.BenchmarkCase.Parameters.PrintInfo,
+                job = r.BenchmarkCase.Job.ResolvedId,
+                name = r.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo,
+                mean = r.Success ? ConvertNanosToMs(r.ResultStatistics.Mean) : 0
+            }).GroupBy(r => r.param);
+
+            int paramCount = 0;
+            foreach (var param in byParam)
+            {
+                var charts = new List<GenericChart.GenericChart>();
+                var jobs = param.GroupBy(p => p.job);
+
+                // Group the legends, then only show the first for each group
+                // https://stackoverflow.com/questions/60751008/sharing-same-legends-for-subplots-in-plotly
+                foreach (var job in jobs)
+                {
+                    var chart2 = Chart2D.Chart
+                        .Column<double, string, string, double, double>(job.Select(j => j.mean), job.Select(j => j.name).ToArray(), Name: job.Key, MarkerColor: colors[job.Key])
+                        .WithLegendGroup(job.Key, paramCount == 0);
+                    charts.Add(chart2);
+                }
+
+                gridCharts.Add(Chart.Combine(charts));
+                paramCount++;
+            }
+
+            // https://github.com/plotly/Plotly.NET/issues/387
+            // The alignment of this isn't 100% correct. Another approach may be to give each sub chart an x axis title
+            double xWidth = 1.0d / byParam.Count();
+            double xMidpoint = xWidth / 2.0d;
+            double[] xs = byParam.Select((_, index) => xMidpoint + (xWidth * index)).ToArray();
+
+            var annotations = byParam.Select((p, index) => Annotation.init<double, double, string, string, string, string, string, string, string, string>(
+                X: xs[index],
+                Y: -0.1,
+                XAnchor: StyleParam.XAnchorPosition.Center,
+                ShowArrow: false,
+                YAnchor: StyleParam.YAnchorPosition.Bottom,
+                Text: p.Key.ToString(),
+                XRef: "paper",
+                YRef: "paper"
+            ));
+
+            // this couples all the charts on the same row to have the same y axis
+            var pattern = new FSharpOption<LayoutGridPattern>(LayoutGridPattern.Coupled);
+
+            Chart
+                .Grid<IEnumerable<GenericChart.GenericChart>>(1, byParam.Count(), Pattern: pattern).Invoke(gridCharts)
+                .WithAnnotations(annotations)
+                .WithoutVerticalGridlines()
+                .WithAxisTitles("Time (ms)")
+                .WithLayout(title)
+                .SaveSVG(file, Width: 1000, Height: 600);
+
+            return new[] { file + ".svg" };
+        }
+
         // internal measurements are in nanos
         // https://github.com/dotnet/BenchmarkDotNet/blob/e4d37d03c0b1ef14e7bde224970bd0fc547fd95a/src/BenchmarkDotNet/Templates/BuildPlots.R#L63-L75
         private static double ConvertNanosToMs(double nanos)
         {
             return nanos * 0.000001;
         }
-
-        public void ExportToLog(Summary summary, ILogger logger)
-        {
-            logger.WriteLine("Generated BarPlot");
-        }
     }
 }

Benchly/BoxPlotExporter.cs

@@ -6,20 +6,20 @@
 
 namespace Benchly
 {
-    // Based on https://github.com/CodeTherapist/BenchmarkDotNetXlsxExporter
     // https://pvk.ca/Blog/2012/07/03/binary-search-star-eliminates-star-branch-mispredictions/
     internal class BoxPlotExporter : IExporter
     {
-        public BoxPlotExporter() 
-        {
-        }
-
         public PlotInfo Info { get; set; } = new PlotInfo();
 
         public string Name => nameof(BoxPlotExporter);
 
         public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
         {
+            if (summary.Reports.Length == 0 || summary.Reports[0].BenchmarkCase.HasParameters)
+            {
+                return Array.Empty<string>();
+            }
+
             var title = this.Info.Title ?? summary.Title;
             var file = Path.Combine(summary.ResultsDirectoryPath, ExporterBase.GetFileName(summary) + "-boxplot");
 

Benchly/PlotExtensions.cs

@@ -47,5 +47,10 @@ public static GenericChart.GenericChart WithGroupBox(this GenericChart.GenericCh
             Layout layout = Layout.init<IConvertible>(BoxMode: BoxMode.Group);
             return chart.WithLayout(layout);
         }
+
+        public static GenericChart.GenericChart WithLegendGroup(this GenericChart.GenericChart chart, string groupName, bool showLegend)
+        {
+            return chart.WithTraceInfo(LegendGroup: new FSharpOption<string>(groupName), ShowLegend: new FSharpOption<bool>(showLegend));
+        }
     }
 }

Benchly/README.md

@@ -1,10 +1,10 @@
-# 📈 benchly 
+# 📊 benchly 
 
 Generate plots for [BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet) using [Plotly.NET](https://github.com/plotly/Plotly.NET/).
 
 # Getting started
 
-Add plot exporter attributes to your benchmark, similar to the built in exporters:
+Add plot exporter attributes to your benchmark:
 
 ```cs
     [BoxPlot(Title = "Box Plot", Colors = "skyblue,slateblue")]
@@ -32,7 +32,7 @@ Add plot exporter attributes to your benchmark, similar to the built in exporter
     }
 ```
 
-Plots are then generated in the results directory when running the benchmarks:
+Plots are written to the results directory after running the benchmarks, like the built in exporters:
 
 ```
   // * Export *