using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using PvpXray; using UnityEditor.PackageManager.ValidationSuite.ValidationTests; using UnityEngine.Profiling; namespace UnityEditor.PackageManager.ValidationSuite { // Attribute for methods to be called before any tests are run, prototype is "void MyMethod(VettingContext)" [AttributeUsage(AttributeTargets.Method)] public class ValidationSuiteSetup : Attribute { } // Attribute for methods to be called after all tests are run, prototype is "void MyMethod(VettingContext)" [AttributeUsage(AttributeTargets.Method)] public class ValidationSuiteTeardown : Attribute { } // Delegate called after every test to provide immediate feedback on single test results. internal delegate void SingleTestCompletedDelegate(IValidationTestResult testResult); // Delegate called after the test run completed, whether it succeeded, failed or got canceled. internal delegate void AllTestsCompletedDelegate(ValidationSuite suite, TestState testRunState); /// /// The validation suite allows you to validate a package while in development /// [InitializeOnLoad] public class ValidationSuite { // List of validation tests private IEnumerable validationTests; // Delegate called after every test to provide immediate feedback on single test results. private SingleTestCompletedDelegate singleTestCompletionDelegate; // Delegate called after the test run completed, whether it succeeded, failed or got canceled. private AllTestsCompletedDelegate allTestsCompletedDelegate; // Vetting context internal readonly VettingContext context; private readonly ValidationSuiteReport report; internal TestState testSuiteState; internal DateTime StartTime; internal DateTime EndTime; public ValidationSuite() { } internal ValidationSuite(SingleTestCompletedDelegate singleTestCompletionDelegate, AllTestsCompletedDelegate allTestsCompletedDelegate, VettingContext context, ValidationSuiteReport report) { this.singleTestCompletionDelegate += singleTestCompletionDelegate; this.allTestsCompletedDelegate += allTestsCompletedDelegate; this.context = context; this.report = report; testSuiteState = TestState.NotRun; BuildTestSuite(); } internal IEnumerable ValidationTests { get { return validationTests.Where(test => (test.SupportedValidations.Contains(context.ValidationType) && test.SupportedPackageTypes.Contains(context.PackageType))); } set { validationTests = value; } } internal IEnumerable ValidationTestResults { get { return validationTests.Cast(); } } /// /// Validate a package for the given validation context. /// /// Package Id in the format of [package name]@[package version]. /// The type of validation to assess. /// True if the validation successfully completed. public static bool ValidatePackage(string packageId, ValidationType validationType) { if (string.IsNullOrEmpty(packageId)) throw new ArgumentNullException(packageId); var packageIdParts = packageId.Split('@'); if (packageIdParts.Length != 2) throw new ArgumentException("Malformed package Id " + packageId); return ValidatePackage(packageIdParts[0], packageIdParts[1], validationType); } /// /// Validate a package for the given validation context. /// /// The name of the package to validate. /// The version of the package to validate. /// The type of validation to assess. /// True if the validation successfully completed. public static bool ValidatePackage(string packageName, string packageVersion, ValidationType validationType) => ValidatePackage(new PackageId(packageName, packageVersion), validationType); internal static bool ValidatePackage(PackageId packageId, ValidationType validationType) { var packagePath = FindPackagePath(packageId.Name); var report = new ValidationSuiteReport(packageId, packagePath); if (string.IsNullOrEmpty(packagePath)) { report.OutputErrorReport(string.Format("Unable to find package \"{0}\" on disk.", packageId.Name)); return false; } // publish locally for embedded and local packages var context = VettingContext.CreatePackmanContext(packageId, validationType); return ValidatePackage(context, out report); } public static bool ValidatePackage(string packageName, string packageVersion, string[] packageIdsForPromotion) => ValidatePackage(new PackageId(packageName, packageVersion), packageIdsForPromotion); static bool ValidatePackage(PackageId packageId, string[] packageIdsForPromotion) { var packagePath = FindPackagePath(packageId.Name); var report = new ValidationSuiteReport(packageId, packagePath); if (string.IsNullOrEmpty(packagePath)) { report.OutputErrorReport($"Unable to find package \"{packageId.Name}\" on disk."); return false; } // publish locally for embedded and local packages var context = VettingContext.CreatePackmanContext(packageId, ValidationType.Promotion); context.packageIdsForPromotion = packageIdsForPromotion; return ValidatePackage(context, out report); } public static bool ValidatePackage(VettingContext context, out ValidationSuiteReport report) { Profiler.BeginSample("ValidatePackage"); report = new ValidationSuiteReport(context.ProjectPackageInfo.PackageId, context.ProjectPackageInfo.path); try { // publish locally for embedded and local packages var testSuite = new ValidationSuite(SingleTestCompletedDelegate, AllTestsCompletedDelegate, context, report); report.Initialize(testSuite.context); testSuite.RunSync(); Profiler.EndSample(); return testSuite.testSuiteState == TestState.Succeeded; } catch (Exception e) { report.OutputErrorReport(string.Format("Test Setup Error: \"{0}\"\r\n", e)); Profiler.EndSample(); return false; } } #if UNITY_2020_1_OR_NEWER [Obsolete("Providing validationType to ValidatePackage has been deprecated, please set validationType in VettingContext (UnityUpgradable) -> !1", false)] #endif public static bool ValidatePackage(VettingContext context, ValidationType validationType, out ValidationSuiteReport report) => ValidatePackage(context, out report); internal static void ValidateEmbeddedPackages(ValidationType validationType) { var packageIdList = new List(); var directories = Directory.GetDirectories("Packages/", "*", SearchOption.TopDirectoryOnly); foreach (var directory in directories) { ActivityLogger.Log("Starting package validation for " + directory); packageIdList.Add(VettingContext.GetManifest(directory).PackageId); } if (packageIdList.Any()) { var success = ValidatePackages(packageIdList, validationType); ActivityLogger.Log("Package validation done and batchmode is set. Shutting down Editor"); EditorApplication.Exit(success ? 0 : 1); } else { EditorApplication.Exit(1); } } internal static bool RunAssetStoreValidationSuite(string packageName, string packageVersion, string packagePath, string previousPackagePath = null) { var packageId = new PackageId(packageName, packageVersion); if (string.IsNullOrEmpty(packagePath)) throw new ArgumentNullException(); var report = new ValidationSuiteReport(packageId, packagePath); try { var context = VettingContext.CreateAssetStoreContext(packageName, packageVersion, packagePath, previousPackagePath); var testSuite = new ValidationSuite(SingleTestCompletedDelegate, AllTestsCompletedDelegate, context, report); testSuite.RunSync(); return testSuite.testSuiteState == TestState.Succeeded; } catch (Exception e) { report.OutputErrorReport(string.Format("\r\nTest Setup Error: \"{0}\"\r\n", e)); return false; } } /// /// Get the validation suite report for the given package. /// /// Package name. /// Package version. /// The validation suite report as a string. public static string GetValidationSuiteReport(string packageName, string packageVersion) { if (string.IsNullOrEmpty(packageName)) throw new ArgumentNullException(packageName); if (string.IsNullOrEmpty(packageVersion)) throw new ArgumentNullException(packageVersion); var packageId = Utilities.CreatePackageId(packageName, packageVersion); return GetValidationSuiteReport(packageId); } /// /// Get the validation suite report for the given package id. /// /// Package Id in the format of [package name]@[package version]. /// The validation suite report as a string. public static string GetValidationSuiteReport(string packageId) { if (string.IsNullOrEmpty(packageId)) throw new ArgumentNullException(packageId); return ValidationSuiteReport.ReportExists(packageId) ? File.ReadAllText(TextReport.ReportPath(packageId)) : null; } internal void RunSync() { Profiler.BeginSample("RunSync"); foreach (var test in validationTests) { test.Context = context; test.Suite = this; Profiler.BeginSample(test.TestName + ".setup"); test.Setup(); Profiler.EndSample(); } Run(); Profiler.EndSample(); } static bool ValidatePackages(IEnumerable packageIds, ValidationType validationType) { var success = true; foreach (var packageId in packageIds) { var result = ValidatePackage(packageId, validationType); if (result) { ActivityLogger.Log("Validation succeeded for " + packageId); } else { success = false; ActivityLogger.Log("Validation failed for " + packageId); } } return success; } void BuildTestSuite() { Profiler.BeginSample("BuildTestSuite"); // Use reflection to discover all Validation Tests in the project with base type == BaseValidation. List testList = new List(); Assembly[] currentDomainAssemblies = AppDomain.CurrentDomain.GetAssemblies(); foreach (Assembly assembly in currentDomainAssemblies) { try { testList.AddRange((from t in Utilities.GetTypesSafe(assembly) where typeof(BaseValidation).IsAssignableFrom(t) && t.GetConstructor(Type.EmptyTypes) != null && !t.IsAbstract select (BaseValidation)Activator.CreateInstance(t)).ToList()); } catch (System.Reflection.ReflectionTypeLoadException) { // There seems to be an isue with assembly.GetTypes throwing an exception. // This quick fix is to allow validation suite to work without blocking anyone // while the owner of this code is contacted. continue; } } validationTests = testList; Profiler.EndSample(); } // Call all static methods with a given attribute type passing them the vetting context void CallSuiteHandler(Type handlerAttributeType) { Profiler.BeginSample("CallSuiteHandler." + handlerAttributeType.Name); foreach (Assembly assembly in AppDomain.CurrentDomain.GetAssemblies()) { try { foreach (Type type in Utilities.GetTypesSafe(assembly)) { foreach (MethodInfo method in type.GetMethods(BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public).Where(methodInfo => methodInfo.GetCustomAttributes(handlerAttributeType, false).Length > 0)) { ParameterInfo[] methodParameters = method.GetParameters(); if ((method.ReturnType != typeof(void)) || (methodParameters.Length != 1) || (methodParameters[0].ParameterType != typeof(VettingContext))) throw new InvalidOperationException("Method '" + type.Name + "." + method.Name + "' with attribute [" + handlerAttributeType.Name + "] has the incorrect prototype, it must be \"void XXX(VettingContext)\""); Profiler.BeginSample("CallSuiteHandler." + type.Name + "." + method.Name); method.Invoke(null, new object[] { context }); Profiler.EndSample(); } } } catch (ReflectionTypeLoadException) { } } Profiler.EndSample(); } void Run() { Profiler.BeginSample("Run"); testSuiteState = TestState.Succeeded; StartTime = DateTime.Now; testSuiteState = TestState.Running; // Let each suite know we are about to start running tests so they can do setup if necessary. CallSuiteHandler(typeof(ValidationSuiteSetup)); // Run through tests foreach (var test in ValidationTests) { if (!test.ShouldRun) continue; try { Profiler.BeginSample(test.TestName + ".run"); test.RunTest(); Profiler.EndSample(); if (test.TestState == TestState.Failed) { testSuiteState = TestState.Failed; } // Signal single test results to caller. singleTestCompletionDelegate(test); } catch (Exception ex) { // if the test didn't behave, return an error. testSuiteState = TestState.Failed; // Change the test outcome. test.AddError(ex.ToString()); singleTestCompletionDelegate(test); } } // Let each suite know we have finished running tests so they can do tidy up if necessary CallSuiteHandler(typeof(ValidationSuiteTeardown)); EndTime = DateTime.Now; if (testSuiteState != TestState.Failed) testSuiteState = TestState.Succeeded; // when we're done, signal the main thread and all other interested allTestsCompletedDelegate(this, testSuiteState); Profiler.EndSample(); } /// /// Find out if the validation suite report exists for the given package id. /// /// Package Id in the format of [package name]@[package version]. /// True if the validation suite report exists. public static bool ReportExists(string packageId) { return ValidationSuiteReport.ReportExists(packageId); } /// /// Find out if the validation suite report exists for the given package id. /// /// Package Id in the format of [package name]@[package version]. /// True if the validation suite report exists. public static bool JsonReportExists(string packageId) { return ValidationSuiteReport.JsonReportExists(packageId); } /// /// Get the validation suite report for the given package id. /// /// Package Id in the format of [package name]@[package version]. /// The validation suite report. public static ValidationSuiteReportData GetReport(string packageId) { return ValidationSuiteReport.GetReport(packageId); } static string FindPackagePath(string packageId) { var path = string.Format("Packages/{0}/package.json", packageId); var absolutePath = Path.GetFullPath(path); return !File.Exists(absolutePath) ? string.Empty : Directory.GetParent(absolutePath).FullName; } static void SingleTestCompletedDelegate(IValidationTestResult testResult) { } static void AllTestsCompletedDelegate(ValidationSuite suite, TestState testRunState) { suite.report.GenerateTextReport(suite); suite.report.GenerateJsonReport(suite); suite.report.GenerateVettingReport(suite); } } }