diff --git a/src/main/scala/ai/privado/cache/RuleCache.scala b/src/main/scala/ai/privado/cache/RuleCache.scala index f5970d3d5..e7588ca77 100644 --- a/src/main/scala/ai/privado/cache/RuleCache.scala +++ b/src/main/scala/ai/privado/cache/RuleCache.scala @@ -32,11 +32,12 @@ import scala.collection.mutable class RuleCache { private var rule: ConfigAndRules = ConfigAndRules(List(), List(), List(), List(), List(), List(), List(), List(), List(), List()) - private val ruleInfoMap = mutable.HashMap[String, RuleInfo]() - private val policyOrThreatMap = mutable.HashMap[String, PolicyOrThreat]() - val internalRules = mutable.HashMap[String, Int]() - val internalPolicies = mutable.Set[String]() - private val storageRuleInfo = mutable.ListBuffer[RuleInfo]() + private val ruleInfoMap = mutable.HashMap[String, RuleInfo]() + private val policyOrThreatMap = mutable.HashMap[String, PolicyOrThreat]() + val internalRules = mutable.HashMap[String, Int]() + val internalPolicies = mutable.Set[String]() + private val storageRuleInfo = mutable.ListBuffer[RuleInfo]() + private val dynamicMergerRuleMap = mutable.HashMap[String, String]() // TODO, rename setRule to withRule as it return the ruleCache object and setters are Unit functions def setRule(rule: ConfigAndRules): RuleCache = { @@ -145,4 +146,16 @@ class RuleCache { } } } + + def addIntoMergedDynamicRuleMapper(externalRuleId: String, internalRuleId: String): Unit = { + dynamicMergerRuleMap(externalRuleId) = internalRuleId + } + + def checkIfMergedDynamicRuleExist(externalRuleId: String): Boolean = { + dynamicMergerRuleMap.contains(externalRuleId) + } + + def getDynamicMappedInternalRule(externalRule: String): String = { + dynamicMergerRuleMap.getOrElse(externalRule, "") + } } diff --git a/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala b/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala index 3dfdfb14e..13f43853a 100644 --- a/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala +++ b/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala @@ -1,5 +1,6 @@ package ai.privado.inputprocessor +import ai.privado.cache.RuleCache import ai.privado.model.{ConfigAndRules, FilterProperty, RuleInfo} import org.slf4j.LoggerFactory @@ -12,10 +13,13 @@ trait DynamicRuleMerger { def mergeDynamicRuleSinkForDependencyDiscovery( externalSinkRules: List[RuleInfo], - internalSinkRules: List[RuleInfo] + internalSinkRules: List[RuleInfo], + ruleCache: RuleCache ): List[RuleInfo] = { try { + val externalOtherRule = new ListBuffer[RuleInfo] + val internalRuleMap = mutable.Map( internalSinkRules .filter(rule => rule.domains.nonEmpty && rule.name.nonEmpty) @@ -23,28 +27,35 @@ trait DynamicRuleMerger { ) externalSinkRules.foreach { externalRule => - val externalDomain = externalRule.domains.headOption.get - val externalRuleName = externalRule.name - val externalFilterProperty = externalRule.filterProperty - - internalRuleMap.collectFirst { - case ((domain, name, filterProperty), rule) - if (domain == externalDomain || name == externalRuleName) && rule.id.contains( - "ThirdParties.SDK" - ) && filterProperty != FilterProperty.CODE => - (domain, name, filterProperty, rule) - } match - case Some(_, _, _, matchingRule: RuleInfo) => - val updatedRule = matchingRule.copy(patterns = matchingRule.patterns ++ externalRule.patterns) - internalRuleMap.update( - (matchingRule.domains.headOption.get, matchingRule.name, matchingRule.filterProperty), - updatedRule - ) - case _ => - internalRuleMap((externalDomain, externalRuleName, externalFilterProperty)) = externalRule + if (externalRule.tags.contains("Discovery_Generated")) { + val externalId = externalRule.id + val externalDomain = externalRule.domains.headOption.get + val externalRuleName = externalRule.name + val externalFilterProperty = externalRule.filterProperty + + internalRuleMap.collectFirst { + case ((domain, name, filterProperty), rule) + if (domain == externalDomain || name == externalRuleName) && rule.id.contains( + "ThirdParties.SDK" + ) && filterProperty != FilterProperty.CODE => + (domain, name, filterProperty, rule) + } match + case Some(_, _, _, matchingRule: RuleInfo) => + val updatedRule = matchingRule.copy(patterns = matchingRule.patterns ++ externalRule.patterns) + internalRuleMap.update( + (matchingRule.domains.headOption.get, matchingRule.name, matchingRule.filterProperty), + updatedRule + ) + ruleCache.addIntoMergedDynamicRuleMapper(externalId, matchingRule.id) + case _ => + internalRuleMap((externalDomain, externalRuleName, externalFilterProperty)) = externalRule + ruleCache.addIntoMergedDynamicRuleMapper(externalRuleName, externalRuleName) + } else { + externalOtherRule.append(externalRule) + } } - internalRuleMap.values.toList + internalRuleMap.values.toList ++ externalOtherRule.toList } catch { case e: Exception => logger.error("Error while merging dynamic rules") diff --git a/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala b/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala index 54fb69ece..db2335ad7 100644 --- a/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala +++ b/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala @@ -313,7 +313,8 @@ trait RuleProcessor extends DynamicRuleMerger { */ val exclusions = externalConfigAndRules.exclusions ++ internalConfigAndRules.exclusions val sources = externalConfigAndRules.sources ++ internalConfigAndRules.sources - val sinks = mergeDynamicRuleSinkForDependencyDiscovery(externalConfigAndRules.sinks, internalConfigAndRules.sinks) + val sinks = + mergeDynamicRuleSinkForDependencyDiscovery(externalConfigAndRules.sinks, internalConfigAndRules.sinks, ruleCache) val collections = externalConfigAndRules.collections ++ internalConfigAndRules.collections val policies = externalConfigAndRules.policies ++ internalConfigAndRules.policies val threats = externalConfigAndRules.threats ++ internalConfigAndRules.threats diff --git a/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala b/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala index 9c99072dd..f26331c8b 100644 --- a/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala +++ b/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala @@ -20,7 +20,11 @@ class DependencyNodeTagger(cpg: Cpg, dependencies: List[DependencyInfo], ruleCac .where(_.file.nameExact(dependency.filePath)) .headOption match { case Some(dep) => - ruleCache.getRuleInfo(dependency.ruleId) match { + val dependencyRuleId = + if (ruleCache.checkIfMergedDynamicRuleExist(dependency.ruleId)) + ruleCache.getDynamicMappedInternalRule(dependency.ruleId) + else dependency.ruleId + ruleCache.getRuleInfo(dependencyRuleId) match { case Some(rule) => Utilities.addRuleTags(builder, dep, rule, ruleCache) case None => logger.error( diff --git a/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala b/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala index 7cf21b37e..050c90592 100644 --- a/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala +++ b/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala @@ -40,7 +40,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { List(".*(software.amazon.awssdk.services.s3).*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -50,9 +50,10 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { ) ) - val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule) + val ruleCache = RuleCache() + val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule, ruleCache) val configAndRule = ConfigAndRules(sinks = finalSinkRule) - val ruleCache = RuleCache().setRule(configAndRule) + ruleCache.setRule(configAndRule) val cpg = code(""" |import software.amazon.awssdk.services.s3.S3Client; @@ -93,7 +94,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { List(".*(software.amazon.awssdk.services.s3).*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -103,9 +104,10 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { ) ) - val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule) + val ruleCache = new RuleCache() + val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule, ruleCache) val configAndRule = ConfigAndRules(sinks = finalSinkRule) - val ruleCache = RuleCache().setRule(configAndRule) + ruleCache.setRule(configAndRule) val cpg = code(""" |import software.amazon.awssdk.services.s3.S3Client; @@ -147,7 +149,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { List(".*(software.amazon.awssdk.services.s3).*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -157,9 +159,10 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { ) ) - val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule) + val ruleCache = new RuleCache() + val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, existingRule, ruleCache) val configAndRule = ConfigAndRules(sinks = finalSinkRule) - val ruleCache = RuleCache().setRule(configAndRule) + ruleCache.setRule(configAndRule) val cpg = code(""" |import software.amazon.awssdk.services.s3.S3Client; @@ -217,7 +220,7 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { List(".*(software.amazon.awssdk.services.s3).*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -227,9 +230,11 @@ class DynamicRuleMergerTest extends JavaFrontendTestSuite, DynamicRuleMerger { ) ) - val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicFilterPropertyRule, existingCodeRule) + val ruleCache = new RuleCache() + val finalSinkRule = + mergeDynamicRuleSinkForDependencyDiscovery(dynamicFilterPropertyRule, existingCodeRule, ruleCache) val configAndRule = ConfigAndRules(sinks = finalSinkRule) - val ruleCache = RuleCache().setRule(configAndRule) + ruleCache.setRule(configAndRule) val cpg = code(""" |import software.amazon.awssdk.services.s3.S3Client; diff --git a/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala b/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala index 445f8995b..987715d7b 100644 --- a/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala @@ -2,7 +2,7 @@ package ai.privado.languageEngine.java.tagger.sink import ai.privado.cache.RuleCache import ai.privado.exporter.SinkExporterValidator -import ai.privado.inputprocessor.DependencyInfo +import ai.privado.inputprocessor.{DependencyInfo, DynamicRuleMerger} import ai.privado.model import ai.privado.model.* import ai.privado.model.exporter.{DataFlowSubCategoryPathExcerptModel, SinkProcessingModel} @@ -11,7 +11,7 @@ import ai.privado.testfixtures.JavaFrontendTestSuite import java.io.File import scala.collection.mutable -class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValidator { +class DependencyTaggerTest extends JavaFrontendTestSuite with DynamicRuleMerger with SinkExporterValidator { "Java pom.xml simple use case" should { val dynamicRule = List( @@ -24,7 +24,7 @@ class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValida List(".*(com.twilio.sdk).*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -41,7 +41,7 @@ class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValida List(".*(com.google.apis).*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -197,7 +197,7 @@ class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValida List("(?i).*com.google.android.gms.*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -214,7 +214,7 @@ class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValida List("(?i).*com.google.firebase.*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -231,7 +231,7 @@ class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValida List("(?i).*com.google.firebase.*"), false, "", - Map(), + Map("Discovery_Generated" -> "true"), NodeType.REGULAR, "", CatLevelOne.SINKS, @@ -314,7 +314,7 @@ class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValida | repositories { | google() | jcenter() - | + | | } |} | @@ -456,4 +456,158 @@ class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValida ) } } + + "Dynamic Merging rule with sinks" should { + val dynamicRule = List( + RuleInfo( + "ThirdParties.SDK.Google", + "Google", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("developers.google.com"), + List("(?i).*com.google.android.gms.*"), + false, + "", + Map("Discovery_Generated" -> "true"), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ) + ) + + val internalRule = List( + RuleInfo( + "ThirdParties.SDK.Google.Play", + "Google", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("developers.google.com"), + List("(?i).*anything.*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ) + ) + + val dependencies = List( + DependencyInfo( + "com.google.android.gms", + "play-services-auth", + "17.0.0", + " implementation 'com.google.android.gms:play-services-auth:17.0.0'\n", + "ThirdParties.SDK.Google", + "Google", + List("developers.google.com"), + List(), + 27, + "app/build.gradle" + ) + ) + + val ruleCache = new RuleCache() + val finalSinkRule = mergeDynamicRuleSinkForDependencyDiscovery(dynamicRule, internalRule, ruleCache) + val configAndRule = ConfigAndRules(sinks = finalSinkRule) + ruleCache.setRule(configAndRule) + + val cpg = code( + """ + |buildscript { + | repositories { + | google() + | jcenter() + | + | } + | dependencies { + | classpath 'com.android.tools.build:gradle:3.5.3' + | classpath 'com.google.gms:google-services:4.2.0' + | + | // NOTE: Do not place your application dependencies here; they belong + | // in the individual module build.gradle files + | } + |} + | + |allprojects { + | repositories { + | google() + | jcenter() + | + | } + |} + | + |task clean(type: Delete) { + | delete rootProject.buildDir + |} + |""".stripMargin, + "build.gradle" + ).moreCode( + """ + |apply plugin: 'com.android.application' + |apply plugin: 'com.google.gms.google-services' + | + |android { + | compileSdkVersion 29 + | buildToolsVersion "29.0.1" + | defaultConfig { + | applicationId "com.example.edu" + | minSdkVersion 22 + | targetSdkVersion 29 + | versionCode 1 + | versionName "1.0" + | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + | } + | buildTypes { + | release { + | minifyEnabled false + | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + | } + | } + |} + | + |dependencies { + | implementation fileTree(dir: 'libs', include: ['*.jar']) + | implementation 'com.google.firebase:firebase-auth:19.0.0' + | implementation 'com.google.android.gms:play-services-auth:17.0.0' + | implementation 'com.github.d-max:spots-dialog:1.1@aar' + | + | testImplementation 'junit:junit:4.12' + | androidTestImplementation 'androidx.test:runner:1.2.0' + | androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + |} + |""".stripMargin, + List("app", "build.gradle").mkString(File.separator) + ).withDependencies(dependencies) + .withRuleCache(ruleCache) + + "3p discovery should get added in processing section" in { + val outputJson = cpg.getPrivadoJson() + getSinks(outputJson).size shouldBe 1 + + val sinksProcessing = getSinkProcessings(outputJson) + + sinksProcessing.headOption.get shouldBe + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Google.Play", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = " implementation 'com.google.android.gms:play-services-auth:17.0.0'\n", + lineNumber = 27, + columnNumber = -1, + fileName = "app/build.gradle", + excerpt = + "}\n\ndependencies {\n implementation fileTree(dir: 'libs', include: ['*.jar'])\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.android.gms:play-services-auth:17.0.0' /* <=== */ \n implementation 'com.github.d-max:spots-dialog:1.1@aar'\n\n testImplementation 'junit:junit:4.12'\n androidTestImplementation 'androidx.test:runner:1.2.0'\n androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'", + arguments = None + ) + ) + ) + } + } }