using NUnit.Framework; using System; using System.Collections.Generic; using System.Diagnostics; using System.Reflection; using UnityEngine; namespace Unity.PerformanceTesting.Benchmark { /// /// Generates and saves a markdown file after running benchmarks. /// public static class BenchmarkGenerator { // This must have the same number of elements as there are bits in the flags parameter for GetFlagSuperscripts static string[] superscripts = { "¹", "²", "³", "⁴", "⁵", "⁶", "⁷", "⁸", "⁹", "¹⁰", "¹¹", "¹²", "¹³", "¹⁴", "¹⁵", "¹⁶", "¹⁷", "¹⁸", "¹⁹", "²⁰", "²¹", "²²", "²³", "²⁴", "²⁵", "²⁶", "²⁷", "²⁸", "²⁹", "³⁰", "³¹", "³²" }; static string[] superscriptDesc = { "Optimizations were disabled to perform this benchmark", "Benchmark run on parallel job workers - results may vary", }; static string GetFlagSuperscripts(uint flags) { string ret = ""; for (int f = 0; f < sizeof(uint) * 8; f++) { if ((flags & (1 << f)) != 0) { if (ret.Length > 0) ret += "˒"; ret += superscripts[f]; } } return ret; } /// /// First, runs benchmarks for all benchmark methods in all types attributed with [Benchmark(benchmarkEnumType)]. /// Then, generates a report in markdown with these results, and saves to the requested file path. /// A common integration method is to call this directly from a menu item handler. /// /// The title of the entire benchmark report /// An enum with a which is specified in all s marking /// classes which contain performance methods to be benchmarked. All performance test methods in the class /// must contain a parameter of the enum marked with which is specified in the class's /// , and may not contain any other parameter with another enum marked with . /// The output file path to save the generated markdown to. /// A global description for the entire benchmark report, or null. /// The title for a global "notes" section for the entire benchmark report, or null. /// An array of notes in the previously mentioned global "notes" section for the entire benchmark report, or null. /// Thrown for any errors in defining the benchmarks. public static void GenerateMarkdown(string title, Type benchmarkEnumType, string filePath, string description = null, string notesTitle = null, string[] notes = null) { var attrBenchmarkComparison = benchmarkEnumType.GetCustomAttribute(); if (attrBenchmarkComparison == null) throw new ArgumentException($"{benchmarkEnumType.Name} is not a valid benchmark comparison enum type as it is not decorated with [{nameof(BenchmarkComparisonAttribute)}]"); Stopwatch timer = new Stopwatch(); timer.Start(); var assemblies = AppDomain.CurrentDomain.GetAssemblies(); var benchmarkTypes = new List(); foreach (Assembly assembly in assemblies) { var types = assembly.GetTypes(); foreach(var t in types) { var cads = t.GetCustomAttributesData(); foreach (var cad in cads) { if (cad.AttributeType != typeof(BenchmarkAttribute)) continue; if ((Type)cad.ConstructorArguments[0].Value == benchmarkEnumType && (bool)cad.ConstructorArguments[1].Value == false) benchmarkTypes.Add(t); } } } UnityEngine.Debug.Log($"Took {timer.Elapsed}s to find all types with [Benchmark(typeof({benchmarkEnumType.Name}))]"); timer.Restart(); GenerateMarkdown(title, benchmarkTypes.ToArray(), filePath, description, notesTitle, notes); UnityEngine.Debug.Log($"Took {timer.Elapsed}s to benchmark all types with [Benchmark(typeof({benchmarkEnumType.Name}))]"); } /// /// First, runs benchmarks for all benchmark methods in all given types.
/// Then, generates a report in markdown with these results, and saves to the requested file path. ///
/// The title of the entire benchmark report /// An array of Types each annotated with a for comparison. Each Type may /// refer to a class with different arguments to the if desired, but all performance test methods in the class /// must each contain a parameter of the enum marked with which is specified in the class's /// , and may not contain any other parameter with another enum marked with . /// The output file path to save the generated markdown to. /// A global description for the entire benchmark report, or null. /// The title for a global "notes" section for the entire benchmark report, or null. /// An array of notes in the previously mentioned global "notes" section for the entire benchmark report, or null. /// Thrown for any errors in defining the benchmarks. public static void GenerateMarkdown(string title, Type[] benchmarkTypes, string filePath, string description = null, string notesTitle = null, string[] notes = null) { using (var reports = BenchmarkRunner.RunBenchmarks(title, benchmarkTypes)) { MarkdownBuilder md = new MarkdownBuilder(); md.Header(1, $"Performance Comparison: {reports.reportName}"); int versionFilter = Application.unityVersion.IndexOf('-'); md.Note($"This file is auto-generated", $"All measurments were taken on {SystemInfo.processorType} with {SystemInfo.processorCount} logical cores.", $"Unity Editor version: {Application.unityVersion.Substring(0, versionFilter == -1 ? Application.unityVersion.Length : versionFilter)}", "To regenerate this file locally use: **DOTS -> Unity.Collections -> Generate ***** menu."); // Generate ToC const string kSectionBenchmarkResults = "Benchmark Results"; md.Header(2, "Table of Contents"); md.ListItem(0).LinkHeader(kSectionBenchmarkResults).Br(); foreach (var group in reports.groups) md.ListItem(1).LinkHeader(group.groupName.ToString()).Br(); // Generate benchmark tables md.Header(2, kSectionBenchmarkResults); // Report description and notes first if (description != null && description.Length > 0) { md.AppendLine(description); md.BrParagraph(); } if (notes != null && notes.Length > 0) { if (notesTitle != null && notesTitle.Length > 0) md.Note(notesTitle, notes); else md.Note(notes); } // Report each group results as ordered in the table of contents foreach (var group in reports.groups) { md.BrParagraph().Header(3, $"*{group.groupName}*"); string[] titles = new string[group.variantNames.Length]; for (int i = 0; i < titles.Length; i++) { titles[i] = group.variantNames[i].ToString(); switch (group.resultTypes[i]) { case BenchmarkResultType.ExternalBaseline: case BenchmarkResultType.External: titles[i] = $"*{titles[i]}*"; break; } } md.TableHeader(false, "Functionality", true, titles); uint tableFlags = 0; // Find max amount of alignment spacing needed int[] ratioSpace = new int[group.variantNames.Length]; foreach (var comparison in group.comparisons) { for (int i = 0; i < ratioSpace.Length; i++) { if (comparison.results[i].ranking == BenchmarkRankingType.Ignored) continue; int ratio10 = Mathf.RoundToInt((float)(comparison.results[i].baselineRatio * 10)); int pow10 = 0; while (ratio10 >= 100) { pow10++; ratio10 /= 10; } ratioSpace[i] = Mathf.Max(ratioSpace[i], pow10); } } foreach (var comparison in group.comparisons) { uint rowFlags = comparison.footnoteFlags; int items = comparison.results.Length; var tableData = new string[items]; for (int i = 0; i < items; i++) { if (comparison.results[i].ranking == BenchmarkRankingType.Ignored) { tableData[i] = "---"; continue; } string format = $"{{0:F{group.resultDecimalPlaces}}}"; string result = $"{string.Format(format, comparison.results[i].Comparator)}{comparison.results[i].UnitSuffix}"; string speedup = $"({comparison.results[i].baselineRatio:F1}x)"; rowFlags |= comparison.results[i].resultFlags; int ratio10 = Mathf.RoundToInt((float)(comparison.results[i].baselineRatio * 10)); if (ratio10 > 10) speedup = $"{speedup}"; else if (ratio10 < 10) speedup = $"{speedup}"; else speedup = $"{speedup}"; int alignSpaces = ratioSpace[i]; while (ratio10 >= 100) { alignSpaces--; ratio10 /= 10; } speedup = $"{new string(' ', alignSpaces)}{speedup}"; tableData[i] = $"{result} {speedup}"; switch (group.resultTypes[i]) { case BenchmarkResultType.ExternalBaseline: case BenchmarkResultType.External: tableData[i] = $"*{tableData[i]}*"; break; } switch (comparison.results[i].ranking) { case BenchmarkRankingType.Normal: tableData[i] = $"{tableData[i]}      "; // those 2 spaces are unicode en-space because >1 ASCII code spaces collapse break; case BenchmarkRankingType.Best: tableData[i] = $"{tableData[i]} 🟢"; break; case BenchmarkRankingType.Worst: tableData[i] = $"{tableData[i]} 🟠"; break; } } tableFlags |= rowFlags; if (rowFlags != 0) md.TableRow($"`{comparison.comparisonName}`*{GetFlagSuperscripts(rowFlags)}*", tableData); else md.TableRow($"`{comparison.comparisonName}`", tableData); } md.Br(); for (int f = 0; f < 32; f++) { if ((tableFlags & (1 << f)) != 0) { if (f < superscriptDesc.Length) md.AppendLine($"*{superscripts[f]}* {superscriptDesc[f]}"); else md.AppendLine($"*{superscripts[f]}* {group.customFootnotes[1u << f]}"); } } md.HorizontalLine(); } md.Save(filePath); } } } }