diff --git a/AppInspector.RulesEngine/AbstractRuleSet.cs b/AppInspector.RulesEngine/AbstractRuleSet.cs index 8e234e77..518e0821 100644 --- a/AppInspector.RulesEngine/AbstractRuleSet.cs +++ b/AppInspector.RulesEngine/AbstractRuleSet.cs @@ -73,62 +73,13 @@ public IEnumerable GetUniversalRules() var expression = new StringBuilder("("); foreach (var pattern in rule.Patterns) { - if (pattern.Pattern != null) + clauses.Add(GenerateClause(pattern, clauseNumber)); + if (clauseNumber > 0) { - var scopes = pattern.Scopes ?? new PatternScope[] { PatternScope.All }; - var modifiers = pattern.Modifiers?.ToList() ?? new List(); - if (pattern.PatternType is PatternType.String or PatternType.Substring) - { - clauses.Add(new OatSubstringIndexClause(scopes, useWordBoundaries: pattern.PatternType == PatternType.String, xPaths: pattern.XPaths, jsonPaths:pattern.JsonPaths) - { - Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification - Data = new List() { pattern.Pattern }, - Capture = true, - Arguments = pattern.Modifiers?.ToList() ?? new List() - }); - if (clauseNumber > 0) - { - expression.Append(" OR "); - } - expression.Append(clauseNumber); - clauseNumber++; - } - else if (pattern.PatternType == PatternType.Regex) - { - clauses.Add(new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths) - { - Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification - Data = new List() { pattern.Pattern }, - Capture = true, - Arguments = modifiers, - CustomOperation = "RegexWithIndex" - }); - if (clauseNumber > 0) - { - expression.Append(" OR "); - } - expression.Append(clauseNumber); - clauseNumber++; - } - else if (pattern.PatternType == PatternType.RegexWord) - { - clauses.Add(new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths) - { - Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification - Data = new List() { $"\\b({pattern.Pattern})\\b" }, - Capture = true, - Arguments = pattern.Modifiers?.ToList() ?? new List(), - CustomOperation = "RegexWithIndex" - }); - - if (clauseNumber > 0) - { - expression.Append(" OR "); - } - expression.Append(clauseNumber); - clauseNumber++; - } + expression.Append(" OR "); } + expression.Append(clauseNumber); + clauseNumber++; } if (clauses.Count > 0) @@ -142,24 +93,43 @@ public IEnumerable GetUniversalRules() foreach (var condition in rule.Conditions ?? Array.Empty()) { - if (condition.Pattern?.Pattern != null) + Clause? clause = GenerateCondition(condition, clauseNumber); + if (clause is { }) + { + clauses.Add(clause); + expression.Append(" AND "); + expression.Append(clauseNumber); + clauseNumber++; + } + } + return new ConvertedOatRule(rule.Id, rule) + { + Clauses = clauses, + Expression = expression.ToString() + }; + } + + private Clause? GenerateCondition(SearchCondition condition, int clauseNumber) + { + if (condition.Pattern is {} conditionPattern) + { + var subClause = GenerateClause(conditionPattern); + if (subClause is null) + { + _logger.LogWarning("SubClause for condition could not be generated"); + } + else { - List conditionModifiers = condition.Pattern.Modifiers?.ToList() ?? new(); if (condition.SearchIn?.Equals("finding-only", StringComparison.InvariantCultureIgnoreCase) != false) { - clauses.Add(new WithinClause() + return new WithinClause() { - Data = new List() { condition.Pattern.Pattern }, Label = clauseNumber.ToString(CultureInfo.InvariantCulture), - Invert = condition.NegateFinding, - Arguments = conditionModifiers, FindingOnly = true, CustomOperation = "Within", - Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All } - }); - expression.Append(" AND "); - expression.Append(clauseNumber); - clauseNumber++; + SubClause = subClause, + Invert = condition.NegateFinding + }; } else if (condition.SearchIn.StartsWith("finding-region", StringComparison.InvariantCultureIgnoreCase)) { @@ -181,93 +151,111 @@ public IEnumerable GetUniversalRules() } if (argList.Count == 2) { - clauses.Add(new WithinClause() + return new WithinClause() { - Data = new List() { condition.Pattern.Pattern }, Label = clauseNumber.ToString(CultureInfo.InvariantCulture), - Invert = condition.NegateFinding, - Arguments = conditionModifiers, FindingRegion = true, CustomOperation = "Within", Before = argList[0], After = argList[1], - Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All } - }); - expression.Append(" AND "); - expression.Append(clauseNumber); - clauseNumber++; + SubClause = subClause, + Invert = condition.NegateFinding + }; } } else if (condition.SearchIn.Equals("same-line", StringComparison.InvariantCultureIgnoreCase)) { - clauses.Add(new WithinClause() + return new WithinClause() { - Data = new List() { condition.Pattern.Pattern }, Label = clauseNumber.ToString(CultureInfo.InvariantCulture), - Invert = condition.NegateFinding, - Arguments = conditionModifiers, SameLineOnly = true, CustomOperation = "Within", - Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All } - }); - expression.Append(" AND "); - expression.Append(clauseNumber); - clauseNumber++; + SubClause = subClause, + Invert = condition.NegateFinding + }; } else if (condition.SearchIn.Equals("same-file", StringComparison.InvariantCultureIgnoreCase)) { - clauses.Add(new WithinClause() + return new WithinClause() { - Data = new List() { condition.Pattern.Pattern }, Label = clauseNumber.ToString(CultureInfo.InvariantCulture), - Invert = condition.NegateFinding, - Arguments = condition.Pattern.Modifiers?.ToList() ?? new List(), - SameFile = true - }); - expression.Append(" AND "); - expression.Append(clauseNumber); - clauseNumber++; + SameFile = true, + SubClause = subClause, + Invert = condition.NegateFinding + }; } else if (condition.SearchIn.Equals("only-before", StringComparison.InvariantCultureIgnoreCase)) { - clauses.Add(new WithinClause() + return new WithinClause() { - Data = new List() { condition.Pattern.Pattern }, Label = clauseNumber.ToString(CultureInfo.InvariantCulture), - Invert = condition.NegateFinding, - Arguments = condition.Pattern.Modifiers?.ToList() ?? new List(), - OnlyBefore = true - }); - expression.Append(" AND "); - expression.Append(clauseNumber); - clauseNumber++; + OnlyBefore = true, + SubClause = subClause, + Invert = condition.NegateFinding + }; } else if (condition.SearchIn.Equals("only-after", StringComparison.InvariantCultureIgnoreCase)) { - clauses.Add(new WithinClause() + return new WithinClause() { - Data = new List() { condition.Pattern.Pattern }, Label = clauseNumber.ToString(CultureInfo.InvariantCulture), - Invert = condition.NegateFinding, - Arguments = condition.Pattern.Modifiers?.ToList() ?? new List(), - OnlyAfter = true - }); - expression.Append(" AND "); - expression.Append(clauseNumber); - clauseNumber++; + OnlyAfter = true, + SubClause = subClause, + Invert = condition.NegateFinding + }; } else { _logger.LogWarning("Search condition {Condition} is not one of the accepted values and this condition will be ignored", condition.SearchIn); - return null; } } + } - return new ConvertedOatRule(rule.Id, rule) + return null; + } + + private Clause? GenerateClause(SearchPattern pattern, int clauseNumber = -1) + { + if (pattern.Pattern != null) { - Clauses = clauses, - Expression = expression.ToString() - }; + var scopes = pattern.Scopes ?? new PatternScope[] { PatternScope.All }; + var modifiers = pattern.Modifiers?.ToList() ?? new List(); + if (pattern.PatternType is PatternType.String or PatternType.Substring) + { + return new OatSubstringIndexClause(scopes, useWordBoundaries: pattern.PatternType == PatternType.String, xPaths: pattern.XPaths, jsonPaths:pattern.JsonPaths) + { + Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification + Data = new List() { pattern.Pattern }, + Capture = true, + Arguments = pattern.Modifiers?.ToList() ?? new List() + }; + } + else if (pattern.PatternType == PatternType.Regex) + { + return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths) + { + Label = clauseNumber.ToString(CultureInfo + .InvariantCulture), //important to pattern index identification + Data = new List() { pattern.Pattern }, + Capture = true, + Arguments = modifiers, + CustomOperation = "RegexWithIndex" + }; + } + else if (pattern.PatternType == PatternType.RegexWord) + { + return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths) + { + Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification + Data = new List() { $"\\b({pattern.Pattern})\\b" }, + Capture = true, + Arguments = pattern.Modifiers?.ToList() ?? new List(), + CustomOperation = "RegexWithIndex" + }; + } + } + + return null; } /// diff --git a/AppInspector.RulesEngine/AppInspector.RulesEngine.csproj b/AppInspector.RulesEngine/AppInspector.RulesEngine.csproj index 8d9afea5..176357f7 100644 --- a/AppInspector.RulesEngine/AppInspector.RulesEngine.csproj +++ b/AppInspector.RulesEngine/AppInspector.RulesEngine.csproj @@ -31,7 +31,7 @@ - + diff --git a/AppInspector.RulesEngine/OatExtensions/GetBoundaryResult.cs b/AppInspector.RulesEngine/OatExtensions/GetBoundaryResult.cs new file mode 100644 index 00000000..b2336173 --- /dev/null +++ b/AppInspector.RulesEngine/OatExtensions/GetBoundaryResult.cs @@ -0,0 +1,17 @@ +namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions; + +internal class GetBoundaryResult +{ + internal GetBoundaryResult(bool validBoundary, int start, int end) + { + ValidBoundary = validBoundary; + Start = start; + End = end; + } + + internal int End { get; } + + internal int Start { get; } + + internal bool ValidBoundary { get; } +} \ No newline at end of file diff --git a/AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexOperation.cs b/AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexOperation.cs index 18cfc29f..ea0de733 100644 --- a/AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexOperation.cs +++ b/AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexOperation.cs @@ -73,6 +73,7 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s if (state1 is TextContainer tc && clause is OatRegexWithIndexClause src && clause.Data is List RegexList && RegexList.Count > 0) { RegexOptions regexOpts = new(); + Boundary? subBoundary = state2 is Boundary s2 ? s2 : null; if (src.Arguments.Contains("i")) { @@ -98,10 +99,9 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s var targets = tc.GetStringFromXPath(xmlPath); foreach (var target in targets) { - var matches = GetMatches(regex, target.Item1, tc, clause, src.Scopes); + var matches = GetMatches(regex, tc, clause, src.Scopes, target.Item2); foreach (var match in matches) { - match.Item2.Index += target.Item2.Index; outmatches.Add(match); } } @@ -114,18 +114,20 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s var targets = tc.GetStringFromJsonPath(jsonPath); foreach (var target in targets) { - var matches = GetMatches(regex, target.Item1, tc, clause, src.Scopes); - foreach (var match in matches) - { - match.Item2.Index += target.Item2.Index; - outmatches.Add(match); - } + outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, target.Item2)); } } } if (src.JsonPaths is null && src.XPaths is null) { - outmatches.AddRange(GetMatches(regex, tc.FullContent, tc, clause, src.Scopes)); + if (subBoundary is not null) + { + outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, subBoundary)); + } + else + { + outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, null)); + } } } } @@ -137,16 +139,16 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s return new OperationResult(false, null); } - private IEnumerable<(int, Boundary)> GetMatches(Regex regex, string content, TextContainer tc, Clause clause, PatternScope[] scopes) + private IEnumerable<(int, Boundary)> GetMatches(Regex regex, TextContainer tc, Clause clause, PatternScope[] scopes, Boundary? boundary) { - foreach (var match in regex.Matches(content)) + foreach (var match in regex.Matches(boundary is null ? tc.FullContent : tc.GetBoundaryText(boundary))) { if (match is Match m) { Boundary translatedBoundary = new() { Length = m.Length, - Index = m.Index + Index = m.Index + (boundary?.Index ?? 0) }; //regex patterns will be indexed off data while string patterns result in N clauses diff --git a/AppInspector.RulesEngine/OatExtensions/OatSubstringIndexOperation.cs b/AppInspector.RulesEngine/OatExtensions/OatSubstringIndexOperation.cs index 3ffc7b72..9f1a7875 100644 --- a/AppInspector.RulesEngine/OatExtensions/OatSubstringIndexOperation.cs +++ b/AppInspector.RulesEngine/OatExtensions/OatSubstringIndexOperation.cs @@ -97,8 +97,16 @@ private OperationResult SubstringIndexOperationDelegate(Clause clause, object? s } if (src.JsonPaths is null && src.XPaths is null) { - var matches = GetMatches(tc.FullContent, stringList[i], comparisonType, tc, src); - outmatches.AddRange(matches.Select(x => (i, x))); + // If state 2 is a boundary, restrict the text provided to check to match the boundary + if (state2 is Boundary boundary) + { + var matches = GetMatches(tc.GetBoundaryText(boundary), stringList[i], comparisonType, tc, src); + outmatches.AddRange(matches.Select(x => (i, x))); } + else + { + var matches = GetMatches(tc.FullContent, stringList[i], comparisonType, tc, src); + outmatches.AddRange(matches.Select(x => (i, x))); } + } } diff --git a/AppInspector.RulesEngine/OatExtensions/WithinClause.cs b/AppInspector.RulesEngine/OatExtensions/WithinClause.cs index 7271531d..2b542bb7 100644 --- a/AppInspector.RulesEngine/OatExtensions/WithinClause.cs +++ b/AppInspector.RulesEngine/OatExtensions/WithinClause.cs @@ -18,7 +18,7 @@ public WithinClause(string? field = null) : base(Operation.Custom, field) public bool SameFile { get; set; } public bool FindingOnly { get; set; } public bool SameLineOnly { get; set; } - public PatternScope[] Scopes { get; set; } = new PatternScope[1] { PatternScope.All }; public bool FindingRegion { get; set; } + public Clause SubClause { get; set; } } } \ No newline at end of file diff --git a/AppInspector.RulesEngine/OatExtensions/WithinOperation.cs b/AppInspector.RulesEngine/OatExtensions/WithinOperation.cs index e03e345a..2c5a6304 100644 --- a/AppInspector.RulesEngine/OatExtensions/WithinOperation.cs +++ b/AppInspector.RulesEngine/OatExtensions/WithinOperation.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -16,6 +17,7 @@ public WithinOperation(Analyzer analyzer, ILoggerFactory? loggerFactory = null) { _loggerFactory = loggerFactory ?? new NullLoggerFactory(); _regexEngine = new RegexOperation(analyzer); + _analyzer = analyzer; CustomOperation = "Within"; OperationDelegate = WithinOperationDelegate; ValidationDelegate = WithinValidationDelegate; @@ -25,159 +27,108 @@ public OperationResult WithinOperationDelegate(Clause c, object? state1, object? { if (c is WithinClause wc && state1 is TextContainer tc) { - var regexOpts = RegexOptions.Compiled; - if (wc.Arguments.Contains("i")) - { - regexOpts |= RegexOptions.IgnoreCase; - } - if (wc.Arguments.Contains("m")) - { - regexOpts |= RegexOptions.Multiline; - } - var passed = new List(); - foreach (var captureHolder in captures ?? Array.Empty()) + List<(int, Boundary)> passed = + new List<(int, Boundary)>(); + List<(int, Boundary)> failed = + new List<(int, Boundary)>(); + + foreach (var capture in captures) { - if (captureHolder is TypedClauseCapture> tcc) + if (capture is TypedClauseCapture> tcc) { - List<(int, Boundary)> toRemove = new(); - foreach ((int clauseNum, Boundary capture) in tcc.Result) + foreach ((int clauseNum, Boundary boundary) in tcc.Result) { - if (wc.FindingOnly) + var boundaryToCheck = GetBoundaryToCheck(); + if (boundaryToCheck is not null) { - var res = ProcessLambda(tc.GetBoundaryText(capture), capture); - if (res.Result) + var operationResult = ProcessLambda(boundaryToCheck); + if (operationResult.Result) { - if (res.Capture is TypedClauseCapture> boundaryList) - { - passed.AddRange(boundaryList.Result); - } + passed.Add((clauseNum, boundary)); } else { - toRemove.Add((clauseNum, capture)); - } + failed.Add((clauseNum, boundary)); + } } - else if (wc.SameLineOnly) + + Boundary? GetBoundaryToCheck() { - var start = tc.LineStarts[tc.GetLocation(capture.Index).Line]; - var end = tc.LineEnds[tc.GetLocation(start + capture.Length).Line]; - var res = ProcessLambda(tc.FullContent[start..end], capture); - if (res.Result) + if (wc.FindingOnly) { - if (res.Capture is TypedClauseCapture> boundaryList) - { - passed.AddRange(boundaryList.Result); - } + return boundary; } - else - { - toRemove.Add((clauseNum, capture)); - } - } - else if (wc.FindingRegion) - { - var startLine = tc.GetLocation(capture.Index).Line; - // Before is already a negative number - var start = tc.LineStarts[Math.Max(1, startLine + wc.Before)]; - var end = tc.LineEnds[Math.Min(tc.LineEnds.Count - 1, startLine + wc.After)]; - var res = ProcessLambda(tc.FullContent[start..(end+1)], capture); - if (res.Result) + if (wc.SameLineOnly) { - if (res.Capture is TypedClauseCapture> boundaryList) + var startInner = tc.LineStarts[tc.GetLocation(boundary.Index).Line]; + var endInner = tc.LineEnds[tc.GetLocation(startInner + boundary.Length).Line]; + return new Boundary() { - passed.AddRange(boundaryList.Result); - } - } - else - { - toRemove.Add((clauseNum, capture)); + Index = startInner, + Length = (endInner - startInner) + 1 + }; } - } - else if (wc.SameFile) - { - var start = tc.LineStarts[0]; - var end = tc.LineEnds[^1]; - var res = ProcessLambda(tc.FullContent[start..end], capture); - if (res.Result) + + if (wc.FindingRegion) { - if (res.Capture is TypedClauseCapture> boundaryList) + var startLine = tc.GetLocation(boundary.Index).Line; + // Before is already a negative number + var startInner = tc.LineStarts[Math.Max(1, startLine + wc.Before)]; + var endInner = tc.LineEnds[Math.Min(tc.LineEnds.Count - 1, startLine + wc.After)]; + var bound = new Boundary() { - passed.AddRange(boundaryList.Result); - } - } - else - { - toRemove.Add((clauseNum, capture)); + Index = startInner, + Length = (endInner - startInner) + 1 + }; + var theText = tc.GetBoundaryText(bound); + return bound; } - } - else if (wc.OnlyBefore) - { - var start = tc.LineStarts[0]; - var end = capture.Index; - var res = ProcessLambda(tc.FullContent[start..end], capture); - if (res.Result) + + if (wc.SameFile) { - if (res.Capture is TypedClauseCapture> boundaryList) + var startInner = tc.LineStarts[0]; + var endInner = tc.LineEnds[^1]; + return new Boundary() { - passed.AddRange(boundaryList.Result); - } + Index = startInner, + Length = (endInner - startInner) + 1 + }; } - else - { - toRemove.Add((clauseNum, capture)); - } } - else if (wc.OnlyAfter) - { - var start = capture.Index + capture.Length; - var end = tc.LineEnds[^1]; - var res = ProcessLambda(tc.FullContent[start..end], capture); - if (res.Result) + + if (wc.OnlyBefore) { - if (res.Capture is TypedClauseCapture> boundaryList) + var startInner = tc.LineStarts[0]; + var endInner = boundary.Index; + return new Boundary() { - passed.AddRange(boundaryList.Result); - } + Index = startInner, + Length = (endInner - startInner) + 1 + }; } - else - { - toRemove.Add((clauseNum, capture)); - } - } - } - tcc.Result.RemoveAll(x => toRemove.Contains(x)); - } - } - // In the case that we have inverted the lambda, the captures are null and thus the passed list will be empty. We thus need to invert this again to get true correctly in that case. - return new OperationResult(passed.Any() ^ wc.Invert, passed.Any() ? new TypedClauseCapture>(wc, passed) : null); - OperationResult ProcessLambda(string target, Boundary targetBoundary) - { - var boundaries = new List(); - foreach (var pattern in wc.Data.Select(x => _regexEngine.StringToRegex(x, regexOpts))) - { - if (pattern is Regex r) - { - var matches = r.Matches(target); - foreach (var match in matches) - { - if (match is Match m) + if (wc.OnlyAfter) { - Boundary translatedBoundary = new() + var startInner = boundary.Index + boundary.Length; + var endInner = tc.LineEnds[^1]; + return new Boundary() { - Length = m.Length, - Index = targetBoundary.Index + m.Index + Index = startInner, + Length = (endInner - startInner) + 1 }; - // Should return only scoped matches - if (tc.ScopeMatch(wc.Scopes, translatedBoundary)) - { - boundaries.Add(translatedBoundary); - } } + + return null; } } } - // Invert the result of the operation if requested - return new OperationResult(boundaries.Any() ^ wc.Invert, boundaries.Any() ? new TypedClauseCapture>(wc, boundaries) : null); + + var passedOrFailed = wc.Invert ? failed : passed; + return new OperationResult(passedOrFailed.Any(), passedOrFailed.Any() ? new TypedClauseCapture>(wc, passedOrFailed.ToList()) : null); + } + + OperationResult ProcessLambda(Boundary target) + { + return _analyzer.GetClauseCapture(wc.SubClause, tc, target, captures); } } return new OperationResult(false, null); @@ -215,18 +166,31 @@ public IEnumerable WithinValidationDelegate(CST.OAT.Rule rule, Clause rule, clause); } } - if (!wc.Data?.Any() ?? true) + if (wc.Data?.Any() ?? false) { - yield return new Violation($"Must provide some regexes as data.", rule, clause); - yield break; + yield return new Violation($"Don't provide data directly. Instead use SubClause..", rule, clause); } + foreach (var datum in wc.Data ?? new List()) { - if (_regexEngine.StringToRegex(datum, RegexOptions.None) is null) + yield return new Violation($"Data in WithinClause is ignored. Use SubClause. Data {datum} found in Rule {rule.Name} Clause {clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture)} is not a valid regex.", rule, clause); + } + + var subOp = _analyzer + .GetOperation(wc.SubClause.Key.Operation, wc.SubClause.Key.CustomOperation); + + if (subOp is null) + { + yield return new Violation($"SubClause in Rule {rule.Name} Clause {clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture)} is of type '{wc.SubClause.Key.Operation},{wc.SubClause.Key.CustomOperation}' is not present in the analyzer.", rule, clause); + } + else + { + foreach (var violation in subOp.ValidationDelegate.Invoke(rule, wc.SubClause)) { - yield return new Violation($"Regex {datum} in Rule {rule.Name} Clause {clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture)} is not a valid regex.", rule, clause); + yield return violation; } } + } else { @@ -236,5 +200,6 @@ public IEnumerable WithinValidationDelegate(CST.OAT.Rule rule, Clause private readonly RegexOperation _regexEngine; private readonly ILoggerFactory _loggerFactory; + private readonly Analyzer _analyzer; } } \ No newline at end of file diff --git a/AppInspector.RulesEngine/RuleProcessor.cs b/AppInspector.RulesEngine/RuleProcessor.cs index 1dabcf02..babfac65 100644 --- a/AppInspector.RulesEngine/RuleProcessor.cs +++ b/AppInspector.RulesEngine/RuleProcessor.cs @@ -143,7 +143,10 @@ public List AnalyzeFile(TextContainer textContainer, FileEntry file var caps = analyzer.GetCaptures(rules, textContainer); foreach (var ruleCapture in caps) { - foreach (var cap in ruleCapture.Captures) + // If we had a WithinClause we only want the captures that passed the within filter. + var filteredCaptures = ruleCapture.Captures.Any(x => x.Clause is WithinClause) + ? ruleCapture.Captures.Where(x => x.Clause is WithinClause) : ruleCapture.Captures; + foreach (var cap in filteredCaptures) { resultsList.AddRange(ProcessBoundary(cap)); } @@ -294,11 +297,14 @@ public async Task> AnalyzeFileAsync(FileEntry fileEntry, Langu TextContainer textContainer = new(await sr.ReadToEndAsync().ConfigureAwait(false), languageInfo.Name, _languages, _opts.LoggerFactory?.CreateLogger() ?? NullLogger.Instance); foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer)) { + // If we had a WithinClause we only want the captures that passed the within filter. + var filteredCaptures = ruleCapture.Captures.Any(x => x.Clause is WithinClause) + ? ruleCapture.Captures.Where(x => x.Clause is WithinClause) : ruleCapture.Captures; if (cancellationToken?.IsCancellationRequested is true) { return resultsList; } - foreach (var cap in ruleCapture.Captures) + foreach (var cap in filteredCaptures) { resultsList.AddRange(ProcessBoundary(cap)); } diff --git a/AppInspector.RulesEngine/TextContainer.cs b/AppInspector.RulesEngine/TextContainer.cs index 2c272d68..939c6841 100644 --- a/AppInspector.RulesEngine/TextContainer.cs +++ b/AppInspector.RulesEngine/TextContainer.cs @@ -56,8 +56,11 @@ public TextContainer(string content, string language, Languages languages, ILogg prefix = languages.GetCommentPrefix(Language); suffix = languages.GetCommentSuffix(Language); inline = languages.GetCommentInline(Language); + Languages = languages; } - + + public Languages Languages { get; set; } + private bool _triedToConstructJsonDocument; private JsonDocument? _jsonDocument; internal IEnumerable<(string, Boundary)> GetStringFromJsonPath(string Path) diff --git a/AppInspector.Tests/DefaultRules/TestDefaultRules.cs b/AppInspector.Tests/DefaultRules/TestDefaultRules.cs index 0c621d36..3b7c3f30 100644 --- a/AppInspector.Tests/DefaultRules/TestDefaultRules.cs +++ b/AppInspector.Tests/DefaultRules/TestDefaultRules.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System; +using System.Diagnostics.CodeAnalysis; using System.Linq; using Microsoft.ApplicationInspector.Commands; using Microsoft.ApplicationInspector.Logging; @@ -26,7 +27,19 @@ public void VerifyDefaultRules() var loggerFactory = new LogOptions() {ConsoleVerbosityLevel = LogEventLevel.Verbose}.GetLoggerFactory(); VerifyRulesCommand command = new(options, loggerFactory); VerifyRulesResult result = command.GetResult(); + foreach (var unverified in result.Unverified) + { + Console.WriteLine("Failed to validate {0}",unverified.RulesId); + foreach (var error in unverified.Errors) + { + Console.WriteLine(error); + } + foreach (var oatError in unverified.OatIssues) + { + Console.WriteLine(oatError.Description); + } + } Assert.AreEqual(VerifyRulesResult.ExitCode.Verified, result.ResultCode); Assert.AreNotEqual(0, result.RuleStatusList.Count); } diff --git a/AppInspector.Tests/RuleProcessor/WithinClauseTests.cs b/AppInspector.Tests/RuleProcessor/WithinClauseTests.cs index 5e7c590f..19afff37 100644 --- a/AppInspector.Tests/RuleProcessor/WithinClauseTests.cs +++ b/AppInspector.Tests/RuleProcessor/WithinClauseTests.cs @@ -121,50 +121,6 @@ public void WithinClauseValidationTests(bool findingOnlySetting, bool findingReg } Assert.AreEqual(expectedNumIssues, verifier.CheckIntegrity(rules).Sum(x => x.OatIssues.Count())); } - - [TestMethod] - public void ValidateRuleHasData() - { - RuleSet rules = new(_loggerFactory); - rules.AddString(validationRule, "TestRules"); - IEnumerable withinClauses = rules - .GetOatRules() - .SelectMany(x => x.Clauses) - .OfType(); - foreach (WithinClause clause in withinClauses) - { - clause.Data = new List(); - } - RulesVerifier verifier = new(new RulesVerifierOptions() {LoggerFactory = _loggerFactory}); - var oatIssues = verifier.CheckIntegrity(rules).SelectMany(x => x.OatIssues); - foreach (var violation in oatIssues) - { - _logger.LogDebug(violation.Description); - } - Assert.AreEqual(1, verifier.CheckIntegrity(rules).Sum(x => x.OatIssues.Count())); - } - - [TestMethod] - public void ValidateRegexAreValid() - { - RuleSet rules = new(_loggerFactory); - rules.AddString(validationRule, "TestRules"); - IEnumerable withinClauses = rules - .GetOatRules() - .SelectMany(x => x.Clauses) - .OfType(); - foreach (WithinClause clause in withinClauses) - { - clause.Data = new List(){"^($"}; - } - RulesVerifier verifier = new(new RulesVerifierOptions() {LoggerFactory = _loggerFactory}); - var oatIssues = verifier.CheckIntegrity(rules).SelectMany(x => x.OatIssues); - foreach (var violation in oatIssues) - { - _logger.LogDebug(violation.Description); - } - Assert.AreEqual(1, verifier.CheckIntegrity(rules).Sum(x => x.OatIssues.Count())); - } [DataRow(true, 1, new[] { 2 })] [DataRow(false, 1, new[] { 3 })] @@ -174,7 +130,7 @@ public void WithinClauseInvertTestForFindingRange0(bool invert, int expectedMatc RuleSet rules = new(_loggerFactory); var newRule = findingRangeZeroRule.Replace("REPLACE_NEGATE", invert.ToString().ToLowerInvariant()); rules.AddString(newRule, "TestRules"); - Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions()); + Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions(){Parallel = false}); if (_languages.FromFileNameOut("test.c", out LanguageInfo info)) { List matches = processor.AnalyzeFile(insideFindingData, new Microsoft.CST.RecursiveExtractor.FileEntry("test.cs", new MemoryStream()), info); @@ -197,7 +153,7 @@ public void MultiLineRegexCondition() { RuleSet rules = new(_loggerFactory); rules.AddString(multiLineRule, "multiline-tests"); - Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions()); + Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions(){Parallel = false}); if (_languages.FromFileNameOut("test.c", out LanguageInfo info)) { List matches = processor.AnalyzeFile(multiLineData, new Microsoft.CST.RecursiveExtractor.FileEntry("test.cs", new MemoryStream()), info); diff --git a/AppInspector/AppInspector.Commands.csproj b/AppInspector/AppInspector.Commands.csproj index cb7b4e5f..8e217dad 100644 --- a/AppInspector/AppInspector.Commands.csproj +++ b/AppInspector/AppInspector.Commands.csproj @@ -54,7 +54,7 @@ - + diff --git a/AppInspector/Commands/VerifyRulesCommand.cs b/AppInspector/Commands/VerifyRulesCommand.cs index 8623d158..abd4f2e0 100644 --- a/AppInspector/Commands/VerifyRulesCommand.cs +++ b/AppInspector/Commands/VerifyRulesCommand.cs @@ -1,6 +1,7 @@ // Copyright (C) Microsoft. All rights reserved. // Licensed under the MIT License. See LICENSE.txt in the project root for license information. +using System.Linq; using Microsoft.ApplicationInspector.RulesEngine.OatExtensions; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -40,6 +41,8 @@ public enum ExitCode [JsonProperty(PropertyName ="ruleStatusList")] public List RuleStatusList { get; set; } + [JsonIgnore] public IEnumerable Unverified => RuleStatusList.Where(x => !x.Verified); + public VerifyRulesResult() { RuleStatusList = new List(); diff --git a/Directory.Build.props b/Directory.Build.props index 690beb44..b9239463 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ - 3.5.108 + 3.5.109 all