Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dynamic Rule Merger Fix #1265

Merged
merged 3 commits into from
Aug 27, 2024
Merged

Dynamic Rule Merger Fix #1265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
29b2a93
added dynamic rule mapper map
ankit-privado Aug 27, 2024
95023f2
remove print statement
ankit-privado Aug 27, 2024
10ad82a
Merge branch 'dev' into third-party-discovery-fix
ankit-privado Aug 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions src/main/scala/ai/privado/cache/RuleCache.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,12 @@ import scala.collection.mutable
class RuleCache {
private var rule: ConfigAndRules =
ConfigAndRules(List(), List(), List(), List(), List(), List(), List(), List(), List(), List())
private val ruleInfoMap = mutable.HashMap[String, RuleInfo]()
private val policyOrThreatMap = mutable.HashMap[String, PolicyOrThreat]()
val internalRules = mutable.HashMap[String, Int]()
val internalPolicies = mutable.Set[String]()
private val storageRuleInfo = mutable.ListBuffer[RuleInfo]()
private val ruleInfoMap = mutable.HashMap[String, RuleInfo]()
private val policyOrThreatMap = mutable.HashMap[String, PolicyOrThreat]()
val internalRules = mutable.HashMap[String, Int]()
val internalPolicies = mutable.Set[String]()
private val storageRuleInfo = mutable.ListBuffer[RuleInfo]()
private val dynamicMergerRuleMap = mutable.HashMap[String, String]()

// TODO, rename setRule to withRule as it return the ruleCache object and setters are Unit functions
def setRule(rule: ConfigAndRules): RuleCache = {
Expand Down Expand Up @@ -145,4 +146,16 @@ class RuleCache {
}
}
}

def addIntoMergedDynamicRuleMapper(externalRuleId: String, internalRuleId: String): Unit = {
dynamicMergerRuleMap(externalRuleId) = internalRuleId
}

def checkIfMergedDynamicRuleExist(externalRuleId: String): Boolean = {
dynamicMergerRuleMap.contains(externalRuleId)
}

def getDynamicMappedInternalRule(externalRule: String): String = {
dynamicMergerRuleMap.getOrElse(externalRule, "")
}
}
53 changes: 32 additions & 21 deletions src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package ai.privado.inputprocessor

import ai.privado.cache.RuleCache
import ai.privado.model.{ConfigAndRules, FilterProperty, RuleInfo}
import org.slf4j.LoggerFactory

Expand All @@ -12,39 +13,49 @@ trait DynamicRuleMerger {

def mergeDynamicRuleSinkForDependencyDiscovery(
externalSinkRules: List[RuleInfo],
internalSinkRules: List[RuleInfo]
internalSinkRules: List[RuleInfo],
ruleCache: RuleCache
): List[RuleInfo] = {
try {

val externalOtherRule = new ListBuffer[RuleInfo]

val internalRuleMap = mutable.Map(
internalSinkRules
.filter(rule => rule.domains.nonEmpty && rule.name.nonEmpty)
.map(rule => ((rule.domains.headOption.get, rule.name, rule.filterProperty), rule)): _*
)

externalSinkRules.foreach { externalRule =>
val externalDomain = externalRule.domains.headOption.get
val externalRuleName = externalRule.name
val externalFilterProperty = externalRule.filterProperty

internalRuleMap.collectFirst {
case ((domain, name, filterProperty), rule)
if (domain == externalDomain || name == externalRuleName) && rule.id.contains(
"ThirdParties.SDK"
) && filterProperty != FilterProperty.CODE =>
(domain, name, filterProperty, rule)
} match
case Some(_, _, _, matchingRule: RuleInfo) =>
val updatedRule = matchingRule.copy(patterns = matchingRule.patterns ++ externalRule.patterns)
internalRuleMap.update(
(matchingRule.domains.headOption.get, matchingRule.name, matchingRule.filterProperty),
updatedRule
)
case _ =>
internalRuleMap((externalDomain, externalRuleName, externalFilterProperty)) = externalRule
if (externalRule.tags.contains("Discovery_Generated")) {
val externalId = externalRule.id
val externalDomain = externalRule.domains.headOption.get
val externalRuleName = externalRule.name
val externalFilterProperty = externalRule.filterProperty

internalRuleMap.collectFirst {
case ((domain, name, filterProperty), rule)
if (domain == externalDomain || name == externalRuleName) && rule.id.contains(
"ThirdParties.SDK"
) && filterProperty != FilterProperty.CODE =>
(domain, name, filterProperty, rule)
} match
case Some(_, _, _, matchingRule: RuleInfo) =>
val updatedRule = matchingRule.copy(patterns = matchingRule.patterns ++ externalRule.patterns)
internalRuleMap.update(
(matchingRule.domains.headOption.get, matchingRule.name, matchingRule.filterProperty),
updatedRule
)
ruleCache.addIntoMergedDynamicRuleMapper(externalId, matchingRule.id)
case _ =>
internalRuleMap((externalDomain, externalRuleName, externalFilterProperty)) = externalRule
ruleCache.addIntoMergedDynamicRuleMapper(externalRuleName, externalRuleName)
} else {
externalOtherRule.append(externalRule)
}
}

internalRuleMap.values.toList
internalRuleMap.values.toList ++ externalOtherRule.toList
} catch {
case e: Exception =>
logger.error("Error while merging dynamic rules")
Expand Down
3 changes: 2 additions & 1 deletion src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ trait RuleProcessor extends DynamicRuleMerger {
*/
val exclusions = externalConfigAndRules.exclusions ++ internalConfigAndRules.exclusions
val sources = externalConfigAndRules.sources ++ internalConfigAndRules.sources
val sinks = mergeDynamicRuleSinkForDependencyDiscovery(externalConfigAndRules.sinks, internalConfigAndRules.sinks)
val sinks =
mergeDynamicRuleSinkForDependencyDiscovery(externalConfigAndRules.sinks, internalConfigAndRules.sinks, ruleCache)
val collections = externalConfigAndRules.collections ++ internalConfigAndRules.collections
val policies = externalConfigAndRules.policies ++ internalConfigAndRules.policies
val threats = externalConfigAndRules.threats ++ internalConfigAndRules.threats
Expand Down
6 changes: 5 additions & 1 deletion src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@ class DependencyNodeTagger(cpg: Cpg, dependencies: List[DependencyInfo], ruleCac
.where(_.file.nameExact(dependency.filePath))
.headOption match {
case Some(dep) =>
ruleCache.getRuleInfo(dependency.ruleId) match {
val dependencyRuleId =
if (ruleCache.checkIfMergedDynamicRuleExist(dependency.ruleId))
ruleCache.getDynamicMappedInternalRule(dependency.ruleId)
else dependency.ruleId
ruleCache.getRuleInfo(dependencyRuleId) match {
case Some(rule) => Utilities.addRuleTags(builder, dep, rule, ruleCache)
case None =>
logger.error(
Expand Down
29 changes: 17 additions & 12 deletions src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
List(".*(software.amazon.awssdk.services.s3).*"),
false,
"",
Map(),
Map("Discovery_Generated" -> "true"),
NodeType.REGULAR,
"",
CatLevelOne.SINKS,
Expand All @@ -50,9 +50,10 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
)
)

val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule)
val ruleCache = RuleCache()
val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule, ruleCache)
val configAndRule = ConfigAndRules(sinks = finalSinkRule)
val ruleCache = RuleCache().setRule(configAndRule)
ruleCache.setRule(configAndRule)

val cpg = code("""
|import software.amazon.awssdk.services.s3.S3Client;
Expand Down Expand Up @@ -93,7 +94,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
List(".*(software.amazon.awssdk.services.s3).*"),
false,
"",
Map(),
Map("Discovery_Generated" -> "true"),
NodeType.REGULAR,
"",
CatLevelOne.SINKS,
Expand All @@ -103,9 +104,10 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
)
)

val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule)
val ruleCache = new RuleCache()
val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule, ruleCache)
val configAndRule = ConfigAndRules(sinks = finalSinkRule)
val ruleCache = RuleCache().setRule(configAndRule)
ruleCache.setRule(configAndRule)

val cpg = code("""
|import software.amazon.awssdk.services.s3.S3Client;
Expand Down Expand Up @@ -147,7 +149,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
List(".*(software.amazon.awssdk.services.s3).*"),
false,
"",
Map(),
Map("Discovery_Generated" -> "true"),
NodeType.REGULAR,
"",
CatLevelOne.SINKS,
Expand All @@ -157,9 +159,10 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
)
)

val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule)
val ruleCache = new RuleCache()
val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule, ruleCache)
val configAndRule = ConfigAndRules(sinks = finalSinkRule)
val ruleCache = RuleCache().setRule(configAndRule)
ruleCache.setRule(configAndRule)

val cpg = code("""
|import software.amazon.awssdk.services.s3.S3Client;
Expand Down Expand Up @@ -217,7 +220,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
List(".*(software.amazon.awssdk.services.s3).*"),
false,
"",
Map(),
Map("Discovery_Generated" -> "true"),
NodeType.REGULAR,
"",
CatLevelOne.SINKS,
Expand All @@ -227,9 +230,11 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger {
)
)

val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicFilterPropertyRule, existingCodeRule)
val ruleCache = new RuleCache()
val finalSinkRule =
mergeDynamicRuleSinkForDependencyDiscovery(dynamicFilterPropertyRule, existingCodeRule, ruleCache)
val configAndRule = ConfigAndRules(sinks = finalSinkRule)
val ruleCache = RuleCache().setRule(configAndRule)
ruleCache.setRule(configAndRule)

val cpg = code("""
|import software.amazon.awssdk.services.s3.S3Client;
Expand Down
Loading
Loading