Skip to content

Commit

Permalink
Refactor Conditions (#495)
Browse files Browse the repository at this point in the history 
* wip checkin

Partial refactor. Requires updated OAT just merged.

* wip checkin

Partial refactor. Requires updated OAT just merged.

* Fix issues with refactored within

* WIP

* Fix inversion in conditions

* Fix regex json rule offset

* Fix off by one

* Validate subclause of within clause.
  • Loading branch information
@gfs
gfs authored Aug 10, 2022
1 parent 175de2a commit dce5493
Show file tree
Hide file tree
Showing 14 changed files with 270 additions and 309 deletions.
218 changes: 103 additions & 115 deletions AppInspector.RulesEngine/AbstractRuleSet.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,62 +73,13 @@ public IEnumerable<ConvertedOatRule> GetUniversalRules()
var expression = new StringBuilder("(");
foreach (var pattern in rule.Patterns)
{
if (pattern.Pattern != null)
clauses.Add(GenerateClause(pattern, clauseNumber));
if (clauseNumber > 0)
{
var scopes = pattern.Scopes ?? new PatternScope[] { PatternScope.All };
var modifiers = pattern.Modifiers?.ToList() ?? new List<string>();
if (pattern.PatternType is PatternType.String or PatternType.Substring)
{
clauses.Add(new OatSubstringIndexClause(scopes, useWordBoundaries: pattern.PatternType == PatternType.String, xPaths: pattern.XPaths, jsonPaths:pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>()
});
if (clauseNumber > 0)
{
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}
else if (pattern.PatternType == PatternType.Regex)
{
clauses.Add(new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = modifiers,
CustomOperation = "RegexWithIndex"
});
if (clauseNumber > 0)
{
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}
else if (pattern.PatternType == PatternType.RegexWord)
{
clauses.Add(new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { $"\\b({pattern.Pattern})\\b" },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>(),
CustomOperation = "RegexWithIndex"
});

if (clauseNumber > 0)
{
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}

if (clauses.Count > 0)
Expand All @@ -142,24 +93,43 @@ public IEnumerable<ConvertedOatRule> GetUniversalRules()

foreach (var condition in rule.Conditions ?? Array.Empty<SearchCondition>())
{
if (condition.Pattern?.Pattern != null)
Clause? clause = GenerateCondition(condition, clauseNumber);
if (clause is { })
{
clauses.Add(clause);
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
}
}
return new ConvertedOatRule(rule.Id, rule)
{
Clauses = clauses,
Expression = expression.ToString()
};
}

private Clause? GenerateCondition(SearchCondition condition, int clauseNumber)
{
if (condition.Pattern is {} conditionPattern)
{
var subClause = GenerateClause(conditionPattern);
if (subClause is null)
{
_logger.LogWarning("SubClause for condition could not be generated");
}
else
{
List<string> conditionModifiers = condition.Pattern.Modifiers?.ToList() ?? new();
if (condition.SearchIn?.Equals("finding-only", StringComparison.InvariantCultureIgnoreCase) != false)
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = conditionModifiers,
FindingOnly = true,
CustomOperation = "Within",
Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All }
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.StartsWith("finding-region", StringComparison.InvariantCultureIgnoreCase))
{
Expand All @@ -181,93 +151,111 @@ public IEnumerable<ConvertedOatRule> GetUniversalRules()
}
if (argList.Count == 2)
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = conditionModifiers,
FindingRegion = true,
CustomOperation = "Within",
Before = argList[0],
After = argList[1],
Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All }
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SubClause = subClause,
Invert = condition.NegateFinding
};
}
}
else if (condition.SearchIn.Equals("same-line", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = conditionModifiers,
SameLineOnly = true,
CustomOperation = "Within",
Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All }
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.Equals("same-file", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = condition.Pattern.Modifiers?.ToList() ?? new List<string>(),
SameFile = true
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SameFile = true,
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.Equals("only-before", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = condition.Pattern.Modifiers?.ToList() ?? new List<string>(),
OnlyBefore = true
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
OnlyBefore = true,
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.Equals("only-after", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = condition.Pattern.Modifiers?.ToList() ?? new List<string>(),
OnlyAfter = true
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
OnlyAfter = true,
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else
{
_logger.LogWarning("Search condition {Condition} is not one of the accepted values and this condition will be ignored", condition.SearchIn);
return null;
}
}

}
return new ConvertedOatRule(rule.Id, rule)
return null;
}

private Clause? GenerateClause(SearchPattern pattern, int clauseNumber = -1)
{
if (pattern.Pattern != null)
{
Clauses = clauses,
Expression = expression.ToString()
};
var scopes = pattern.Scopes ?? new PatternScope[] { PatternScope.All };
var modifiers = pattern.Modifiers?.ToList() ?? new List<string>();
if (pattern.PatternType is PatternType.String or PatternType.Substring)
{
return new OatSubstringIndexClause(scopes, useWordBoundaries: pattern.PatternType == PatternType.String, xPaths: pattern.XPaths, jsonPaths:pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>()
};
}
else if (pattern.PatternType == PatternType.Regex)
{
return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo
.InvariantCulture), //important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = modifiers,
CustomOperation = "RegexWithIndex"
};
}
else if (pattern.PatternType == PatternType.RegexWord)
{
return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { $"\\b({pattern.Pattern})\\b" },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>(),
CustomOperation = "RegexWithIndex"
};
}
}

return null;
}

/// <summary>
Expand Down
2 changes: 1 addition & 1 deletion AppInspector.RulesEngine/AppInspector.RulesEngine.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@

<ItemGroup>
<PackageReference Include="JsonCons.JsonPath" Version="1.1.0" />
<PackageReference Include="Microsoft.CST.OAT" Version="1.2.19" />
<PackageReference Include="Microsoft.CST.OAT" Version="1.2.24" />
<PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.11" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
Expand Down
17 changes: 17 additions & 0 deletions AppInspector.RulesEngine/OatExtensions/GetBoundaryResult.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions;

internal class GetBoundaryResult
{
internal GetBoundaryResult(bool validBoundary, int start, int end)
{
ValidBoundary = validBoundary;
Start = start;
End = end;
}

internal int End { get; }

internal int Start { get; }

internal bool ValidBoundary { get; }
}
26 changes: 14 additions & 12 deletions AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexOperation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s
if (state1 is TextContainer tc && clause is OatRegexWithIndexClause src && clause.Data is List<string> RegexList && RegexList.Count > 0)
{
RegexOptions regexOpts = new();
Boundary? subBoundary = state2 is Boundary s2 ? s2 : null;

if (src.Arguments.Contains("i"))
{
Expand All @@ -98,10 +99,9 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s
var targets = tc.GetStringFromXPath(xmlPath);
foreach (var target in targets)
{
var matches = GetMatches(regex, target.Item1, tc, clause, src.Scopes);
var matches = GetMatches(regex, tc, clause, src.Scopes, target.Item2);
foreach (var match in matches)
{
match.Item2.Index += target.Item2.Index;
outmatches.Add(match);
}
}
Expand All @@ -114,18 +114,20 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s
var targets = tc.GetStringFromJsonPath(jsonPath);
foreach (var target in targets)
{
var matches = GetMatches(regex, target.Item1, tc, clause, src.Scopes);
foreach (var match in matches)
{
match.Item2.Index += target.Item2.Index;
outmatches.Add(match);
}
outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, target.Item2));
}
}
}
if (src.JsonPaths is null && src.XPaths is null)
{
outmatches.AddRange(GetMatches(regex, tc.FullContent, tc, clause, src.Scopes));
if (subBoundary is not null)
{
outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, subBoundary));
}
else
{
outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, null));
}
}
}
}
Expand All @@ -137,16 +139,16 @@ private OperationResult RegexWithIndexOperationDelegate(Clause clause, object? s
return new OperationResult(false, null);
}

private IEnumerable<(int, Boundary)> GetMatches(Regex regex, string content, TextContainer tc, Clause clause, PatternScope[] scopes)
private IEnumerable<(int, Boundary)> GetMatches(Regex regex, TextContainer tc, Clause clause, PatternScope[] scopes, Boundary? boundary)
{
foreach (var match in regex.Matches(content))
foreach (var match in regex.Matches(boundary is null ? tc.FullContent : tc.GetBoundaryText(boundary)))
{
if (match is Match m)
{
Boundary translatedBoundary = new()
{
Length = m.Length,
Index = m.Index
Index = m.Index + (boundary?.Index ?? 0)
};

//regex patterns will be indexed off data while string patterns result in N clauses
Expand Down
Loading

0 comments on commit dce5493

Please sign in to comment.