diff --git a/AppInspector.RulesEngine/RuleStatus.cs b/AppInspector.RulesEngine/RuleStatus.cs index 74ad6eed..239f2db7 100644 --- a/AppInspector.RulesEngine/RuleStatus.cs +++ b/AppInspector.RulesEngine/RuleStatus.cs @@ -11,4 +11,6 @@ public class RuleStatus public bool Verified => !Errors.Any() && !OatIssues.Any(); public IEnumerable Errors { get; set; } = Enumerable.Empty(); public IEnumerable OatIssues { get; set; } = Enumerable.Empty(); + public bool HasPositiveSelfTests { get; set; } + public bool HasNegativeSelfTests { get; set; } } \ No newline at end of file diff --git a/AppInspector.RulesEngine/RulesVerifier.cs b/AppInspector.RulesEngine/RulesVerifier.cs index f681dd87..1f0d8357 100644 --- a/AppInspector.RulesEngine/RulesVerifier.cs +++ b/AppInspector.RulesEngine/RulesVerifier.cs @@ -235,7 +235,7 @@ public RuleStatus CheckIntegrity(ConvertedOatRule convertedOatRule) // Then we fall back to grab any language from the languages configuration that isn't DoesNotApplyTo for this rule. var language = convertedOatRule.AppInspectorRule.AppliesTo?.FirstOrDefault() ?? _options.LanguageSpecs.GetNames().FirstOrDefault(x => !convertedOatRule.AppInspectorRule.DoesNotApplyTo?.Contains(x, StringComparer.InvariantCultureIgnoreCase) ?? true) ?? "csharp"; - + // validate all must match samples are matched foreach (var mustMatchElement in rule.MustMatch ?? Array.Empty()) { @@ -246,7 +246,7 @@ public RuleStatus CheckIntegrity(ConvertedOatRule convertedOatRule) errors.Add($"Rule {rule.Id} does not match the 'MustMatch' test {mustMatchElement}. "); } } - + // validate no must not match conditions are matched foreach (var mustNotMatchElement in rule.MustNotMatch ?? Array.Empty()) { @@ -257,18 +257,21 @@ public RuleStatus CheckIntegrity(ConvertedOatRule convertedOatRule) errors.Add($"Rule {rule.Id} matches the 'MustNotMatch' test '{mustNotMatchElement}'."); } } - + if (rule.Tags?.Length == 0) { _logger?.LogError("Rule must specify tags. {0}", rule.Id); errors.Add($"Rule must specify tags. {rule.Id}"); } + return new RuleStatus() { RulesId = rule.Id, RulesName = rule.Name, Errors = errors, - OatIssues = _analyzer.EnumerateRuleIssues(convertedOatRule) + OatIssues = _analyzer.EnumerateRuleIssues(convertedOatRule), + HasPositiveSelfTests = rule.MustMatch?.Length > 0, + HasNegativeSelfTests = rule.MustNotMatch?.Length > 0 }; } } diff --git a/AppInspector.Tests/DefaultRules/TestDefaultRules.cs b/AppInspector.Tests/DefaultRules/TestDefaultRules.cs index 3b7c3f30..59c1584c 100644 --- a/AppInspector.Tests/DefaultRules/TestDefaultRules.cs +++ b/AppInspector.Tests/DefaultRules/TestDefaultRules.cs @@ -4,6 +4,7 @@ using Microsoft.ApplicationInspector.Commands; using Microsoft.ApplicationInspector.Logging; using Microsoft.ApplicationInspector.RulesEngine; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Serilog.Events; @@ -24,22 +25,46 @@ public void VerifyDefaultRules() { VerifyDefaultRules = true, }; - var loggerFactory = new LogOptions() {ConsoleVerbosityLevel = LogEventLevel.Verbose}.GetLoggerFactory(); + var loggerFactory = new LogOptions() {ConsoleVerbosityLevel = LogEventLevel.Debug}.GetLoggerFactory(); + var logger = loggerFactory.CreateLogger("Tests"); VerifyRulesCommand command = new(options, loggerFactory); VerifyRulesResult result = command.GetResult(); + + if (result.Unverified.Any()) + { + logger.Log(LogLevel.Error, "{0} of {1} rules failed verification. Errors are as follows:", result.Unverified.Count(), result.RuleStatusList.Count); + } + else + { + logger.Log(LogLevel.Information, "All {0} rules passed validation.", result.RuleStatusList.Count); + } + foreach (var unverified in result.Unverified) { - Console.WriteLine("Failed to validate {0}",unverified.RulesId); + logger.Log(LogLevel.Error, "Failed to validate {0}",unverified.RulesId); foreach (var error in unverified.Errors) { - Console.WriteLine(error); + logger.Log(LogLevel.Error, error); } foreach (var oatError in unverified.OatIssues) { - Console.WriteLine(oatError.Description); + logger.Log(LogLevel.Error, oatError.Description); } } + + logger.Log(LogLevel.Information, "{0} of {1} rules have positive self tests.",result.RuleStatusList.Count(x => x.HasPositiveSelfTests),result.RuleStatusList.Count); + logger.Log(LogLevel.Information, "{0} of {1} rules have negative self tests.",result.RuleStatusList.Count(x => x.HasNegativeSelfTests),result.RuleStatusList.Count); + + foreach (var rule in result.RuleStatusList.Where(x => !x.HasPositiveSelfTests)) + { + logger.Log(LogLevel.Warning, "Rule {0} does not have any positive test cases", rule.RulesId); + } + foreach (var rule in result.RuleStatusList.Where(x => !x.HasNegativeSelfTests)) + { + logger.Log(LogLevel.Warning, "Rule {0} does not have any negative test cases", rule.RulesId); + } + Assert.AreEqual(VerifyRulesResult.ExitCode.Verified, result.ResultCode); Assert.AreNotEqual(0, result.RuleStatusList.Count); }