using System; using System.Collections.Generic; using Unity.PerformanceTesting.Data; using Unity.PerformanceTesting.Runtime; using Unity.PerformanceTesting.Exceptions; using Unity.PerformanceTesting.Meters; using Unity.PerformanceTesting.Statistics; using UnityEngine; using UnityEngine.Profiling; namespace Unity.PerformanceTesting.Measurements { /// /// Used as a helper class to sample execution time of methods. Uses fluent pattern to build and needs to be executed with Run method. /// public class MethodMeasurement { internal const int k_MeasurementCount = 9; private const int k_MinMeasurementTimeMs = 100; private const int k_MinWarmupTimeMs = 100; private const int k_ProbingMultiplier = 4; private const int k_MaxIterations = 10000; internal const int k_MaxDynamicMeasurements = 1000; private const double k_DefaultMaxRelativeError = 0.02; private const ConfidenceLevel k_DefaultConfidenceLevel = ConfidenceLevel.L99; private const OutlierMode k_DefaultOutlierMode = OutlierMode.Remove; private readonly Action m_Action; private readonly List m_SampleGroups = new List(); private readonly Recorder m_GCRecorder; private Action m_Setup; private Action m_Cleanup; private SampleGroup m_SampleGroup = new SampleGroup("Time", SampleUnit.Millisecond, false); private SampleGroup m_SampleGroupGC = new SampleGroup("Time.GC()", SampleUnit.Undefined, false); private int m_WarmupCount; private int m_MeasurementCount; internal bool m_DynamicMeasurementCount; private double m_MaxRelativeError = k_DefaultMaxRelativeError; private ConfidenceLevel m_ConfidenceLevel = k_DefaultConfidenceLevel; private OutlierMode m_OutlierMode = k_DefaultOutlierMode; private int m_IterationCount = 1; private bool m_GC; private IStopWatch m_Watch; /// /// Initializes a method measurement. /// /// Method to be measured. public MethodMeasurement(Action action) { m_Action = action; m_GCRecorder = Recorder.Get("GC.Alloc"); m_GCRecorder.enabled = false; if (m_Watch == null) m_Watch = new StopWatch(); } internal MethodMeasurement StopWatch(IStopWatch watch) { m_Watch = watch; return this; } /// /// Will record provided profiler markers once per frame. /// /// Profiler marker names as in profiler window. /// public MethodMeasurement ProfilerMarkers(params string[] profilerMarkerNames) { if (profilerMarkerNames == null) return this; foreach (var marker in profilerMarkerNames) { var sampleGroup = new SampleGroup(marker, SampleUnit.Nanosecond, false); sampleGroup.GetRecorder(); sampleGroup.Recorder.enabled = false; m_SampleGroups.Add(sampleGroup); } return this; } /// /// Will record provided profiler markers once per frame with additional control over the SampleUnit. /// /// List of SampleGroups where a name matches the profiler marker and desired SampleUnit. /// public MethodMeasurement ProfilerMarkers(params SampleGroup[] sampleGroups) { if (sampleGroups == null){ return this;} foreach (var sampleGroup in sampleGroups) { sampleGroup.GetRecorder(); sampleGroup.Recorder.enabled = false; m_SampleGroups.Add(sampleGroup); } return this; } /// /// Overrides the default SampleGroup of "Time". /// /// Desired name for measurement SampleGroup. /// public MethodMeasurement SampleGroup(string name) { m_SampleGroup = new SampleGroup(name, SampleUnit.Millisecond, false); m_SampleGroupGC = new SampleGroup(name + ".GC()", SampleUnit.Undefined, false); return this; } /// /// Overrides the default SampleGroup. /// /// SampleGroup with your desired name and unit. /// public MethodMeasurement SampleGroup(SampleGroup sampleGroup) { m_SampleGroup = sampleGroup; m_SampleGroupGC = new SampleGroup(sampleGroup.Name + ".GC()", SampleUnit.Undefined, false); return this; } /// /// Count of times to execute before measurements are collected. If unspecified, a default warmup will be assigned. /// /// Count of warmup iterations to execute. /// public MethodMeasurement WarmupCount(int count) { m_WarmupCount = count; return this; } /// /// Specifies the amount of method executions for a single measurement. /// /// Count of method executions. /// public MethodMeasurement IterationsPerMeasurement(int count) { m_IterationCount = count; return this; } /// /// Specifies the number of measurements to take. /// /// Count of measurements to take. /// public MethodMeasurement MeasurementCount(int count) { m_MeasurementCount = count; return this; } /// /// Dynamically find a suitable measurement count based on the margin of error of the samples. /// The measurements will stop once a certain amount of samples (specified by a confidence interval) /// falls within an acceptable error range from the result (defined by a relative error of the mean). /// A default margin of error range of 2% and a default confidence interval of 99% will be used. /// /// Outlier mode allows to include or exclude outliers when evaluating the stop criterion. /// public MethodMeasurement DynamicMeasurementCount(OutlierMode outlierMode = k_DefaultOutlierMode) { m_DynamicMeasurementCount = true; m_OutlierMode = outlierMode; return this; } /// /// Dynamically find a suitable measurement count based on the margin of error of the samples. /// The measurements will stop once a certain amount of samples (specified by a confidence interval) /// falls within an acceptable error range from the result (defined by a relative error of the mean). /// /// The maximum relative error of the mean that the margin of error must fall into. /// The confidence interval which will be used to calculate the margin of error. /// Outlier mode allows to include or exclude outliers when evaluating the stop criterion. /// public MethodMeasurement DynamicMeasurementCount(double maxRelativeError, ConfidenceLevel confidenceLevel = k_DefaultConfidenceLevel, OutlierMode outlierMode = k_DefaultOutlierMode) { m_MaxRelativeError = maxRelativeError; m_ConfidenceLevel = confidenceLevel; m_DynamicMeasurementCount = true; m_OutlierMode = outlierMode; return this; } /// /// Used to provide a cleanup method which will not be measured. /// /// Cleanup method to execute. /// public MethodMeasurement CleanUp(Action action) { m_Cleanup = action; return this; } /// /// Used to provide a setup method which will run before the measurement. /// /// Setup method to execute. /// public MethodMeasurement SetUp(Action action) { m_Setup = action; return this; } /// /// Enables recording of garbage collector calls. /// /// public MethodMeasurement GC() { m_GC = true; return this; } /// /// Executes the measurement with given parameters. When MeasurementCount is not provided, a probing method will run to determine desired measurement counts. /// public void Run() { ValidateCorrectDynamicMeasurementCountUsage(); SettingsOverride(); var settingsCount = RunSettings.Instance.MeasurementCount; if (m_MeasurementCount > 0 || settingsCount > -1) { Warmup(m_WarmupCount); RunForIterations(m_IterationCount, m_MeasurementCount, useAverage: false); return; } if (m_DynamicMeasurementCount) { Warmup(m_WarmupCount); RunForIterations(m_IterationCount); return; } var iterations = Probing(); RunForIterations(iterations, k_MeasurementCount, useAverage: true); } private void ValidateCorrectDynamicMeasurementCountUsage() { if (!m_DynamicMeasurementCount) return; if (m_MeasurementCount > 0) { m_DynamicMeasurementCount = false; Debug.LogWarning("DynamicMeasurementCount will be ignored because MeasurementCount was specified."); } } /// /// Overrides measurement count based on performance run settings /// private void SettingsOverride() { var count = RunSettings.Instance.MeasurementCount; if (count < 0) { return; } m_MeasurementCount = count; m_WarmupCount = m_WarmupCount > 0 ? count : 0; m_DynamicMeasurementCount = false; } private void RunForIterations(int iterations, int measurements, bool useAverage) { EnableMarkers(); for (var j = 0; j < measurements; j++) { var executionTime = iterations == 1 ? ExecuteSingleIteration() : ExecuteForIterations(iterations); if (useAverage) executionTime /= iterations; var delta = Utils.ConvertSample(SampleUnit.Millisecond, m_SampleGroup.Unit, executionTime); Measure.Custom(m_SampleGroup, delta); } DisableAndMeasureMarkers(); } private void RunForIterations(int iterations) { EnableMarkers(); while(true) { var executionTime = iterations == 1 ? ExecuteSingleIteration() : ExecuteForIterations(iterations); var delta = Utils.ConvertSample(SampleUnit.Millisecond, m_SampleGroup.Unit, executionTime); Measure.Custom(m_SampleGroup, delta); if (SampleCountFulfillsRequirements()) break; } DisableAndMeasureMarkers(); } private void EnableMarkers() { foreach (var sampleGroup in m_SampleGroups) { sampleGroup.Recorder.enabled = true; } } private void DisableAndMeasureMarkers() { foreach (var sampleGroup in m_SampleGroups) { sampleGroup.Recorder.enabled = false; var sample = sampleGroup.Recorder.elapsedNanoseconds; var blockCount = sampleGroup.Recorder.sampleBlockCount; if(blockCount == 0) continue; var delta = Utils.ConvertSample(SampleUnit.Nanosecond, sampleGroup.Unit, sample); Measure.Custom(sampleGroup, delta / blockCount); } } private bool SampleCountFulfillsRequirements() { var samples = m_SampleGroup.Samples; var sampleCount = samples.Count; var statistics = MeasurementsStatistics.Calculate(samples, m_OutlierMode, m_ConfidenceLevel); var actualError = statistics.MarginOfError; var maxError = m_MaxRelativeError * statistics.Mean; if (sampleCount >= k_MeasurementCount && actualError < maxError) return true; if (sampleCount >= k_MaxDynamicMeasurements) return true; return false; } private int Probing() { var executionTime = 0.0D; var iterations = 1; if (m_WarmupCount > 0) throw new PerformanceTestException( "Please provide MeasurementCount or remove WarmupCount in your usage of Measure.Method"); while (executionTime < k_MinWarmupTimeMs) { executionTime = m_Watch.Split(); Warmup(iterations); executionTime = m_Watch.Split() - executionTime; if (executionTime < k_MinWarmupTimeMs) { iterations *= k_ProbingMultiplier; } } if (iterations == 1) { ExecuteActionWithCleanupSetup(); ExecuteActionWithCleanupSetup(); return 1; } var deisredIterationsCount = Mathf.Clamp((int) (k_MinMeasurementTimeMs * iterations / executionTime), 1, k_MaxIterations); return deisredIterationsCount; } private void Warmup(int iterations) { for (var i = 0; i < iterations; i++) { ExecuteForIterations(m_IterationCount); } } private double ExecuteActionWithCleanupSetup() { m_Setup?.Invoke(); var executionTime = m_Watch.Split(); m_Action.Invoke(); executionTime = m_Watch.Split() - executionTime; m_Cleanup?.Invoke(); return executionTime; } private double ExecuteSingleIteration() { if (m_GC) StartGCRecorder(); m_Setup?.Invoke(); var executionTime = m_Watch.Split(); m_Action.Invoke(); executionTime = m_Watch.Split() - executionTime; m_Cleanup?.Invoke(); if (m_GC) EndGCRecorderAndMeasure(1); return executionTime; } private double ExecuteForIterations(int iterations) { if (m_GC) StartGCRecorder(); var executionTime = 0.0D; if (m_Cleanup != null || m_Setup != null) { for (var i = 0; i < iterations; i++) { executionTime += ExecuteActionWithCleanupSetup(); } } else { executionTime = m_Watch.Split(); for (var i = 0; i < iterations; i++) { m_Action.Invoke(); } executionTime = m_Watch.Split() - executionTime; } if (m_GC) EndGCRecorderAndMeasure(iterations); return executionTime; } private void StartGCRecorder() { System.GC.Collect(); m_GCRecorder.enabled = false; m_GCRecorder.enabled = true; } private void EndGCRecorderAndMeasure(int iterations) { m_GCRecorder.enabled = false; Measure.Custom(m_SampleGroupGC, (double) m_GCRecorder.sampleBlockCount / iterations); } } }