Show only original source in processing for data element (#761)

* add - show only original source in processing for data element

* added test case

* merged dev

* fixed a failing test case

* fix - remove derived source

* Revert "Type recovery changes (#785)"

This reverts commit c534728.

* Revert "Revert "Type recovery changes (#785)""

This reverts commit 3a7f22c.

* Update README.md

README.md

@@ -1,7 +1,7 @@
 Privado Core
 =============================================
 
-Branch structure
+Branch structure 
 
 main - This branch will contain the released version of the code.
 

src/main/scala/ai/privado/exporter/JSONExporter.scala

@@ -25,6 +25,7 @@ package ai.privado.exporter
 
 import ai.privado.audit.AuditReportEntryPoint.DataElementDiscoveryAudit
 import ai.privado.cache.{AppCache, DataFlowCache, Environment, RuleCache, TaggerCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.metric.MetricHandler
 import ai.privado.model.Constants.{outputDirectoryName, value}
 import ai.privado.model.exporter.{
@@ -74,16 +75,18 @@ object JSONExporter {
     dataflows: Map[String, Path],
     ruleCache: RuleCache,
     taggerCache: TaggerCache = new TaggerCache(),
-    dataFlowCache: DataFlowCache
+    dataFlowCache: DataFlowCache,
+    privadoInput: PrivadoInput
   ): Either[String, Unit] = {
     logger.info("Initiated exporter engine")
-    val sourceExporter          = new SourceExporter(cpg, ruleCache)
-    val sinkExporter            = new SinkExporter(cpg, ruleCache)
-    val dataflowExporter        = new DataflowExporter(cpg, dataflows, taggerCache, dataFlowCache)
-    val collectionExporter      = new CollectionExporter(cpg, ruleCache)
-    val probableSinkExporter    = new ProbableSinkExporter(cpg, ruleCache, repoPath)
-    val policyAndThreatExporter = new PolicyAndThreatExporter(cpg, ruleCache, dataflows, taggerCache, dataFlowCache)
-    val output                  = mutable.LinkedHashMap[String, Json]()
+    val sourceExporter       = new SourceExporter(cpg, ruleCache, privadoInput)
+    val sinkExporter         = new SinkExporter(cpg, ruleCache)
+    val dataflowExporter     = new DataflowExporter(cpg, dataflows, taggerCache, dataFlowCache)
+    val collectionExporter   = new CollectionExporter(cpg, ruleCache)
+    val probableSinkExporter = new ProbableSinkExporter(cpg, ruleCache, repoPath)
+    val policyAndThreatExporter =
+      new PolicyAndThreatExporter(cpg, ruleCache, dataflows, taggerCache, dataFlowCache, privadoInput)
+    val output = mutable.LinkedHashMap[String, Json]()
     try {
 
       output.addOne(Constants.coreVersion -> Environment.privadoVersionCore.asJson)

src/main/scala/ai/privado/exporter/PolicyAndThreatExporter.scala

@@ -24,6 +24,7 @@
 package ai.privado.exporter
 
 import ai.privado.cache.{AppCache, DataFlowCache, RuleCache, TaggerCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.languageEngine.java.threatEngine.ThreatEngineExecutor
 import ai.privado.model.exporter.{ViolationDataFlowModel, ViolationModel, ViolationProcessingModel}
 import ai.privado.policyEngine.PolicyExecutor
@@ -39,14 +40,16 @@ class PolicyAndThreatExporter(
   ruleCache: RuleCache,
   dataflows: Map[String, Path],
   taggerCache: TaggerCache,
-  dataFlowCache: DataFlowCache
+  dataFlowCache: DataFlowCache,
+  privadoInput: PrivadoInput
 ) {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
   def getViolations(repoPath: String): List[ViolationModel] = {
-    val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache)
-    val threatExecutor = new ThreatEngineExecutor(cpg, dataflows, repoPath, ruleCache, taggerCache, dataFlowCache)
+    val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput)
+    val threatExecutor =
+      new ThreatEngineExecutor(cpg, dataflows, repoPath, ruleCache, taggerCache, dataFlowCache, privadoInput)
 
     try {
       threatExecutor.getProcessingViolations(ruleCache.getAllThreat) ++ policyExecutor.getProcessingViolations

src/main/scala/ai/privado/exporter/SourceExporter.scala

@@ -24,19 +24,19 @@
 package ai.privado.exporter
 
 import ai.privado.cache.RuleCache
-import ai.privado.entrypoint.ScanProcessor
+import ai.privado.entrypoint.{PrivadoInput, ScanProcessor}
 import ai.privado.model.exporter.{SourceModel, SourceProcessingModel}
 import ai.privado.model.{CatLevelOne, Constants, InternalTag}
 import ai.privado.utility.Utilities
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Tag}
-import ai.privado.semantic.Language._
-import io.shiftleft.semanticcpg.language._
+import ai.privado.semantic.Language.*
+import io.shiftleft.semanticcpg.language.*
 import overflowdb.traversal.Traversal
 
 import scala.collection.mutable
 
-class SourceExporter(cpg: Cpg, ruleCache: RuleCache) {
+class SourceExporter(cpg: Cpg, ruleCache: RuleCache, privadoInput: PrivadoInput) {
 
   lazy val sourcesList: List[AstNode]      = getSourcesList
   lazy val sourcesTagList: List[List[Tag]] = sourcesList.map(_.tag.l)
@@ -67,7 +67,7 @@ class SourceExporter(cpg: Cpg, ruleCache: RuleCache) {
           entrySet._1,
           ExporterUtility
             .convertPathElements({
-              if (ScanProcessor.config.disableDeDuplication)
+              if (privadoInput.disableDeDuplication)
                 entrySet._2.toList
               else
                 entrySet._2.toList
@@ -84,9 +84,12 @@ class SourceExporter(cpg: Cpg, ruleCache: RuleCache) {
     */
   private def getSourcesList: List[AstNode] = {
     def filterSource(traversal: Traversal[AstNode]) = {
-      traversal.tag
-        .nameExact(Constants.catLevelOne)
-        .or(_.valueExact(CatLevelOne.SOURCES.name), _.valueExact(CatLevelOne.DERIVED_SOURCES.name))
+      traversal
+        .where(
+          _.tag
+            .nameExact(Constants.catLevelOne)
+            .valueExact(CatLevelOne.SOURCES.name)
+        )
     }
     val sources =
       cpg.identifier

src/main/scala/ai/privado/languageEngine/default/processor/DefaultProcessor.scala

@@ -100,7 +100,8 @@ object DefaultProcessor {
             dataflowMap,
             ruleCache,
             taggerCache,
-            dataFlowCache
+            dataFlowCache,
+            ScanProcessor.config
           ) match {
             case Left(err) =>
               MetricHandler.otherErrorsOrWarnings.addOne(err)

src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala

@@ -87,7 +87,8 @@ object GoProcessor {
             dataflowMap,
             ruleCache,
             taggerCache,
-            dataFlowCache
+            dataFlowCache,
+            ScanProcessor.config
           ) match {
             case Left(err) =>
               MetricHandler.otherErrorsOrWarnings.addOne(err)

src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala

@@ -122,7 +122,8 @@ object JavaProcessor {
             dataflowMap,
             ruleCache,
             taggerCache,
-            dataFlowCache
+            dataFlowCache,
+            ScanProcessor.config
           ) match {
             case Left(err) =>
               MetricHandler.otherErrorsOrWarnings.addOne(err)

src/main/scala/ai/privado/languageEngine/java/tagger/source/IdentifierTagger.scala

@@ -131,7 +131,7 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache)
             })
 
           // To Mark all field Access and getters
-          tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMemberName)
+          tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMemberName, true)
         })
       })
 
@@ -181,7 +181,7 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache)
       })
 
       // To Mark all field Access and getters
-      tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMember.name)
+      tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMember.name, true)
     })
   }
 
@@ -210,7 +210,7 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache)
         .get(typeDecl.fullName)
         .flatMap(_.get(ruleInfo.id))
       // To Mark all field Access and getters
-      tagAllFieldAccessAndGetters(builder, typeDecl.fullName, ruleInfo, membersOption.map(_.name).mkString("|"))
+      tagAllFieldAccessAndGetters(builder, typeDecl.fullName, ruleInfo, membersOption.map(_.name).mkString("|"), true)
 
     })
 
@@ -272,14 +272,17 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache)
     builder: BatchedUpdate.DiffGraphBuilder,
     typeDeclVal: String,
     ruleInfo: RuleInfo,
-    typeDeclMemberName: String
+    typeDeclMemberName: String,
+    isDerived: Boolean = false
   ): Unit = {
     val impactedGetters = getFieldAccessCallsMatchingRegex(cpg, typeDeclVal, s"($typeDeclMemberName)")
       .filterNot(item => item.code.equals(item.code.toUpperCase))
 
     impactedGetters.foreach(impactedGetter => {
       storeForTag(builder, impactedGetter, ruleCache)(InternalTag.SENSITIVE_FIELD_ACCESS.toString)
       addRuleTags(builder, impactedGetter, ruleInfo, ruleCache)
+      if (isDerived)
+        storeForTag(builder, impactedGetter, ruleCache)(Constants.catLevelOne, CatLevelOne.DERIVED_SOURCES.name)
     })
 
     val impactedReturnMethods = getCallsMatchingReturnRegex(cpg, typeDeclVal, s"($typeDeclMemberName)")

src/main/scala/ai/privado/languageEngine/java/threatEngine/CookieConsentMgmtModule.scala

@@ -1,6 +1,7 @@
 package ai.privado.languageEngine.java.threatEngine
 
 import ai.privado.cache.{AppCache, DataFlowCache, RuleCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.languageEngine.java.threatEngine.ThreatUtility.hasDataElements
 import ai.privado.model.PolicyOrThreat
 import ai.privado.model.exporter.ViolationProcessingModel
@@ -27,10 +28,11 @@ object CookieConsentMgmtModule {
     cpg: Cpg,
     dataflows: Map[String, Path],
     ruleCache: RuleCache,
-    dataFlowCache: DataFlowCache
+    dataFlowCache: DataFlowCache,
+    privadoInput: PrivadoInput
   ): Try[(Boolean, List[ViolationProcessingModel])] = Try {
     if (hasDataElements(cpg)) {
-      val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache)
+      val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput)
       val violatingFlows = policyExecutor.getViolatingOccurrencesForPolicy(threat)
 
       val consentMgmtModulePresent      = cpg.call.methodFullName(getCookieConsentMgmtModulePattern(threat.config))

src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToLogs.scala

@@ -24,6 +24,7 @@
 package ai.privado.languageEngine.java.threatEngine
 
 import ai.privado.cache.{AppCache, DataFlowCache, RuleCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.model.exporter.ViolationDataFlowModel
 import ai.privado.policyEngine.PolicyExecutor
 import io.shiftleft.codepropertygraph.generated.Cpg
@@ -50,13 +51,14 @@ object DataLeakageToLogs {
     cpg: Cpg,
     dataflows: Map[String, Path],
     ruleCache: RuleCache,
-    dataFlowCache: DataFlowCache
+    dataFlowCache: DataFlowCache,
+    privadoInput: PrivadoInput
   ): Try[(Boolean, List[ViolationDataFlowModel])] = Try {
     // use policy executor to directly process existing flows
     // we already have this implementation as part of policy enforcement
     // threat being type of suggestive policy
     // might restructure this in future and have central utilities consumed by both
-    val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache)
+    val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput)
     val violatingFlows = policyExecutor.getViolatingFlowsForPolicy(threat)
 
     // violation if empty

src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToNotifications.scala

@@ -24,6 +24,7 @@
 package ai.privado.languageEngine.java.threatEngine
 
 import ai.privado.cache.{AppCache, DataFlowCache, RuleCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.model.exporter.ViolationDataFlowModel
 import ai.privado.model.PolicyOrThreat
 import ai.privado.policyEngine.PolicyExecutor
@@ -50,13 +51,14 @@ object DataLeakageToNotifications {
     cpg: Cpg,
     dataflows: Map[String, Path],
     ruleCache: RuleCache,
-    dataFlowCache: DataFlowCache
+    dataFlowCache: DataFlowCache,
+    privadoInput: PrivadoInput
   ): Try[(Boolean, List[ViolationDataFlowModel])] = Try {
     // use policy executor to directly process existing flows (we have rule for notifications)
     // we already have this implementation as part of policy enforcement
     // threat being type of suggestive policy
     // might restructure this in future and have central utilities consumed by both
-    val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache)
+    val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput)
     val violatingFlows = policyExecutor.getViolatingFlowsForPolicy(threat)
 
     // violation if empty

src/main/scala/ai/privado/languageEngine/java/threatEngine/ThreatEngineExecutor.scala

@@ -24,6 +24,7 @@
 package ai.privado.languageEngine.java.threatEngine
 
 import ai.privado.cache.{DataFlowCache, RuleCache, TaggerCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.exporter.ExporterUtility
 import ai.privado.model.exporter.ViolationModel
 import ai.privado.model.PolicyOrThreat
@@ -40,7 +41,8 @@ class ThreatEngineExecutor(
   repoPath: String,
   ruleCache: RuleCache,
   taggerCache: TaggerCache,
-  dataFlowCache: DataFlowCache
+  dataFlowCache: DataFlowCache,
+  privadoInput: PrivadoInput
 ) {
 
   private val logger = LoggerFactory.getLogger(getClass)
@@ -128,7 +130,7 @@ class ThreatEngineExecutor(
         }
 
       case "PrivadoPolicy.CookieConsent.IsCookieConsentMgmtModuleImplemented" =>
-        CookieConsentMgmtModule.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache) match {
+        CookieConsentMgmtModule.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache, privadoInput) match {
           case Success(res) => Some(res)
           case Failure(e) => {
             logger.debug(s"Error for ${threatId}: ${e}")
@@ -221,15 +223,15 @@ class ThreatEngineExecutor(
 
     val violationResponse = threatId match {
       case "Threats.Sharing.isDataExposedToThirdPartiesViaNotification" if isAndroidRepo =>
-        DataLeakageToNotifications.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache) match {
+        DataLeakageToNotifications.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache, privadoInput) match {
           case Success(res) => Some(res)
           case Failure(e) => {
             logger.debug(s"Error for ${threatId}: ${e}")
             None
           }
         }
       case "Threats.Leakage.isDataLeakingToLog" =>
-        DataLeakageToLogs.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache) match {
+        DataLeakageToLogs.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache, privadoInput) match {
           case Success(res) => Some(res)
           case Failure(e) => {
             logger.debug(s"Error for ${threatId}: ${e}")

src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala

@@ -100,7 +100,8 @@ object JavascriptProcessor {
           dataflowMap,
           ruleCache,
           taggerCache,
-          dataFlowCache
+          dataFlowCache,
+          ScanProcessor.config
         ) match {
           case Left(err) =>
             MetricHandler.otherErrorsOrWarnings.addOne(err)

src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala

@@ -116,7 +116,8 @@ object PythonProcessor {
             dataflowMap,
             ruleCache,
             taggerCache,
-            dataFlowCache
+            dataFlowCache,
+            ScanProcessor.config
           ) match {
             case Left(err) =>
               MetricHandler.otherErrorsOrWarnings.addOne(err)

src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala

@@ -189,7 +189,8 @@ object RubyProcessor {
             dataflowMap,
             ruleCache,
             taggerCache,
-            dataFlowCache
+            dataFlowCache,
+            ScanProcessor.config
           ) match {
             case Left(err) =>
               MetricHandler.otherErrorsOrWarnings.addOne(err)

src/main/scala/ai/privado/policyEngine/PolicyExecutor.scala

@@ -23,6 +23,7 @@
 package ai.privado.policyEngine
 
 import ai.privado.cache.{DataFlowCache, RuleCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.exporter.ExporterUtility
 import ai.privado.languageEngine.java.threatEngine.ThreatUtility.getSourceNode
 import ai.privado.model.exporter.{ViolationDataFlowModel, ViolationProcessingModel}
@@ -38,7 +39,13 @@ import overflowdb.traversal.Traversal
 import scala.collection.mutable
 import scala.util.{Failure, Success, Try}
 
-class PolicyExecutor(cpg: Cpg, dataFlowCache: DataFlowCache, repoName: String, ruleCache: RuleCache) {
+class PolicyExecutor(
+  cpg: Cpg,
+  dataFlowCache: DataFlowCache,
+  repoName: String,
+  ruleCache: RuleCache,
+  privadoInput: PrivadoInput
+) {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
@@ -52,7 +59,7 @@ class PolicyExecutor(cpg: Cpg, dataFlowCache: DataFlowCache, repoName: String, r
   // Map to contain sinkId -> List(pathIds)
   lazy val dataflowSinkIdMap: Map[String, List[String]] = getDataflowBySinkIdMapping
 
-  val sourceExporter = new SourceExporter(cpg, ruleCache)
+  val sourceExporter = new SourceExporter(cpg, ruleCache, privadoInput)
 
   lazy val sourceExporterModel = sourceExporter.getSources