diff --git a/README.md b/README.md index cee43bbda..020a1f0b8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Privado Core ============================================= -Branch structure +Branch structure main - This branch will contain the released version of the code. diff --git a/src/main/scala/ai/privado/exporter/JSONExporter.scala b/src/main/scala/ai/privado/exporter/JSONExporter.scala index 46830c429..73023c526 100644 --- a/src/main/scala/ai/privado/exporter/JSONExporter.scala +++ b/src/main/scala/ai/privado/exporter/JSONExporter.scala @@ -25,6 +25,7 @@ package ai.privado.exporter import ai.privado.audit.AuditReportEntryPoint.DataElementDiscoveryAudit import ai.privado.cache.{AppCache, DataFlowCache, Environment, RuleCache, TaggerCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.metric.MetricHandler import ai.privado.model.Constants.{outputDirectoryName, value} import ai.privado.model.exporter.{ @@ -74,16 +75,18 @@ object JSONExporter { dataflows: Map[String, Path], ruleCache: RuleCache, taggerCache: TaggerCache = new TaggerCache(), - dataFlowCache: DataFlowCache + dataFlowCache: DataFlowCache, + privadoInput: PrivadoInput ): Either[String, Unit] = { logger.info("Initiated exporter engine") - val sourceExporter = new SourceExporter(cpg, ruleCache) - val sinkExporter = new SinkExporter(cpg, ruleCache) - val dataflowExporter = new DataflowExporter(cpg, dataflows, taggerCache, dataFlowCache) - val collectionExporter = new CollectionExporter(cpg, ruleCache) - val probableSinkExporter = new ProbableSinkExporter(cpg, ruleCache, repoPath) - val policyAndThreatExporter = new PolicyAndThreatExporter(cpg, ruleCache, dataflows, taggerCache, dataFlowCache) - val output = mutable.LinkedHashMap[String, Json]() + val sourceExporter = new SourceExporter(cpg, ruleCache, privadoInput) + val sinkExporter = new SinkExporter(cpg, ruleCache) + val dataflowExporter = new DataflowExporter(cpg, dataflows, taggerCache, dataFlowCache) + val collectionExporter = new CollectionExporter(cpg, ruleCache) + val probableSinkExporter = new ProbableSinkExporter(cpg, ruleCache, repoPath) + val policyAndThreatExporter = + new PolicyAndThreatExporter(cpg, ruleCache, dataflows, taggerCache, dataFlowCache, privadoInput) + val output = mutable.LinkedHashMap[String, Json]() try { output.addOne(Constants.coreVersion -> Environment.privadoVersionCore.asJson) diff --git a/src/main/scala/ai/privado/exporter/PolicyAndThreatExporter.scala b/src/main/scala/ai/privado/exporter/PolicyAndThreatExporter.scala index 2748ed964..723d587d4 100644 --- a/src/main/scala/ai/privado/exporter/PolicyAndThreatExporter.scala +++ b/src/main/scala/ai/privado/exporter/PolicyAndThreatExporter.scala @@ -24,6 +24,7 @@ package ai.privado.exporter import ai.privado.cache.{AppCache, DataFlowCache, RuleCache, TaggerCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.languageEngine.java.threatEngine.ThreatEngineExecutor import ai.privado.model.exporter.{ViolationDataFlowModel, ViolationModel, ViolationProcessingModel} import ai.privado.policyEngine.PolicyExecutor @@ -39,14 +40,16 @@ class PolicyAndThreatExporter( ruleCache: RuleCache, dataflows: Map[String, Path], taggerCache: TaggerCache, - dataFlowCache: DataFlowCache + dataFlowCache: DataFlowCache, + privadoInput: PrivadoInput ) { private val logger = LoggerFactory.getLogger(getClass) def getViolations(repoPath: String): List[ViolationModel] = { - val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache) - val threatExecutor = new ThreatEngineExecutor(cpg, dataflows, repoPath, ruleCache, taggerCache, dataFlowCache) + val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput) + val threatExecutor = + new ThreatEngineExecutor(cpg, dataflows, repoPath, ruleCache, taggerCache, dataFlowCache, privadoInput) try { threatExecutor.getProcessingViolations(ruleCache.getAllThreat) ++ policyExecutor.getProcessingViolations diff --git a/src/main/scala/ai/privado/exporter/SourceExporter.scala b/src/main/scala/ai/privado/exporter/SourceExporter.scala index 4bad3fc5a..ab5ab55aa 100644 --- a/src/main/scala/ai/privado/exporter/SourceExporter.scala +++ b/src/main/scala/ai/privado/exporter/SourceExporter.scala @@ -24,19 +24,19 @@ package ai.privado.exporter import ai.privado.cache.RuleCache -import ai.privado.entrypoint.ScanProcessor +import ai.privado.entrypoint.{PrivadoInput, ScanProcessor} import ai.privado.model.exporter.{SourceModel, SourceProcessingModel} import ai.privado.model.{CatLevelOne, Constants, InternalTag} import ai.privado.utility.Utilities import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Tag} -import ai.privado.semantic.Language._ -import io.shiftleft.semanticcpg.language._ +import ai.privado.semantic.Language.* +import io.shiftleft.semanticcpg.language.* import overflowdb.traversal.Traversal import scala.collection.mutable -class SourceExporter(cpg: Cpg, ruleCache: RuleCache) { +class SourceExporter(cpg: Cpg, ruleCache: RuleCache, privadoInput: PrivadoInput) { lazy val sourcesList: List[AstNode] = getSourcesList lazy val sourcesTagList: List[List[Tag]] = sourcesList.map(_.tag.l) @@ -67,7 +67,7 @@ class SourceExporter(cpg: Cpg, ruleCache: RuleCache) { entrySet._1, ExporterUtility .convertPathElements({ - if (ScanProcessor.config.disableDeDuplication) + if (privadoInput.disableDeDuplication) entrySet._2.toList else entrySet._2.toList @@ -84,9 +84,12 @@ class SourceExporter(cpg: Cpg, ruleCache: RuleCache) { */ private def getSourcesList: List[AstNode] = { def filterSource(traversal: Traversal[AstNode]) = { - traversal.tag - .nameExact(Constants.catLevelOne) - .or(_.valueExact(CatLevelOne.SOURCES.name), _.valueExact(CatLevelOne.DERIVED_SOURCES.name)) + traversal + .where( + _.tag + .nameExact(Constants.catLevelOne) + .valueExact(CatLevelOne.SOURCES.name) + ) } val sources = cpg.identifier diff --git a/src/main/scala/ai/privado/languageEngine/default/processor/DefaultProcessor.scala b/src/main/scala/ai/privado/languageEngine/default/processor/DefaultProcessor.scala index 6a59e3632..5e123643e 100644 --- a/src/main/scala/ai/privado/languageEngine/default/processor/DefaultProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/default/processor/DefaultProcessor.scala @@ -100,7 +100,8 @@ object DefaultProcessor { dataflowMap, ruleCache, taggerCache, - dataFlowCache + dataFlowCache, + ScanProcessor.config ) match { case Left(err) => MetricHandler.otherErrorsOrWarnings.addOne(err) diff --git a/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala b/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala index f33fb1a1a..fc4490c91 100644 --- a/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala @@ -87,7 +87,8 @@ object GoProcessor { dataflowMap, ruleCache, taggerCache, - dataFlowCache + dataFlowCache, + ScanProcessor.config ) match { case Left(err) => MetricHandler.otherErrorsOrWarnings.addOne(err) diff --git a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala index 9b4f9eb8a..4c4d6496f 100644 --- a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala @@ -122,7 +122,8 @@ object JavaProcessor { dataflowMap, ruleCache, taggerCache, - dataFlowCache + dataFlowCache, + ScanProcessor.config ) match { case Left(err) => MetricHandler.otherErrorsOrWarnings.addOne(err) diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/source/IdentifierTagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/source/IdentifierTagger.scala index d817c9cba..94d03f744 100644 --- a/src/main/scala/ai/privado/languageEngine/java/tagger/source/IdentifierTagger.scala +++ b/src/main/scala/ai/privado/languageEngine/java/tagger/source/IdentifierTagger.scala @@ -131,7 +131,7 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache) }) // To Mark all field Access and getters - tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMemberName) + tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMemberName, true) }) }) @@ -181,7 +181,7 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache) }) // To Mark all field Access and getters - tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMember.name) + tagAllFieldAccessAndGetters(builder, typeDeclVal, ruleInfo, typeDeclMember.name, true) }) } @@ -210,7 +210,7 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache) .get(typeDecl.fullName) .flatMap(_.get(ruleInfo.id)) // To Mark all field Access and getters - tagAllFieldAccessAndGetters(builder, typeDecl.fullName, ruleInfo, membersOption.map(_.name).mkString("|")) + tagAllFieldAccessAndGetters(builder, typeDecl.fullName, ruleInfo, membersOption.map(_.name).mkString("|"), true) }) @@ -272,7 +272,8 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache) builder: BatchedUpdate.DiffGraphBuilder, typeDeclVal: String, ruleInfo: RuleInfo, - typeDeclMemberName: String + typeDeclMemberName: String, + isDerived: Boolean = false ): Unit = { val impactedGetters = getFieldAccessCallsMatchingRegex(cpg, typeDeclVal, s"($typeDeclMemberName)") .filterNot(item => item.code.equals(item.code.toUpperCase)) @@ -280,6 +281,8 @@ class IdentifierTagger(cpg: Cpg, ruleCache: RuleCache, taggerCache: TaggerCache) impactedGetters.foreach(impactedGetter => { storeForTag(builder, impactedGetter, ruleCache)(InternalTag.SENSITIVE_FIELD_ACCESS.toString) addRuleTags(builder, impactedGetter, ruleInfo, ruleCache) + if (isDerived) + storeForTag(builder, impactedGetter, ruleCache)(Constants.catLevelOne, CatLevelOne.DERIVED_SOURCES.name) }) val impactedReturnMethods = getCallsMatchingReturnRegex(cpg, typeDeclVal, s"($typeDeclMemberName)") diff --git a/src/main/scala/ai/privado/languageEngine/java/threatEngine/CookieConsentMgmtModule.scala b/src/main/scala/ai/privado/languageEngine/java/threatEngine/CookieConsentMgmtModule.scala index e6b6de0ca..6e1bde1d5 100644 --- a/src/main/scala/ai/privado/languageEngine/java/threatEngine/CookieConsentMgmtModule.scala +++ b/src/main/scala/ai/privado/languageEngine/java/threatEngine/CookieConsentMgmtModule.scala @@ -1,6 +1,7 @@ package ai.privado.languageEngine.java.threatEngine import ai.privado.cache.{AppCache, DataFlowCache, RuleCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.languageEngine.java.threatEngine.ThreatUtility.hasDataElements import ai.privado.model.PolicyOrThreat import ai.privado.model.exporter.ViolationProcessingModel @@ -27,10 +28,11 @@ object CookieConsentMgmtModule { cpg: Cpg, dataflows: Map[String, Path], ruleCache: RuleCache, - dataFlowCache: DataFlowCache + dataFlowCache: DataFlowCache, + privadoInput: PrivadoInput ): Try[(Boolean, List[ViolationProcessingModel])] = Try { if (hasDataElements(cpg)) { - val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache) + val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput) val violatingFlows = policyExecutor.getViolatingOccurrencesForPolicy(threat) val consentMgmtModulePresent = cpg.call.methodFullName(getCookieConsentMgmtModulePattern(threat.config)) diff --git a/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToLogs.scala b/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToLogs.scala index e93b04e78..127343f91 100644 --- a/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToLogs.scala +++ b/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToLogs.scala @@ -24,6 +24,7 @@ package ai.privado.languageEngine.java.threatEngine import ai.privado.cache.{AppCache, DataFlowCache, RuleCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.model.exporter.ViolationDataFlowModel import ai.privado.policyEngine.PolicyExecutor import io.shiftleft.codepropertygraph.generated.Cpg @@ -50,13 +51,14 @@ object DataLeakageToLogs { cpg: Cpg, dataflows: Map[String, Path], ruleCache: RuleCache, - dataFlowCache: DataFlowCache + dataFlowCache: DataFlowCache, + privadoInput: PrivadoInput ): Try[(Boolean, List[ViolationDataFlowModel])] = Try { // use policy executor to directly process existing flows // we already have this implementation as part of policy enforcement // threat being type of suggestive policy // might restructure this in future and have central utilities consumed by both - val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache) + val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput) val violatingFlows = policyExecutor.getViolatingFlowsForPolicy(threat) // violation if empty diff --git a/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToNotifications.scala b/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToNotifications.scala index 1f4fb64d3..712d87fcd 100644 --- a/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToNotifications.scala +++ b/src/main/scala/ai/privado/languageEngine/java/threatEngine/DataLeakageToNotifications.scala @@ -24,6 +24,7 @@ package ai.privado.languageEngine.java.threatEngine import ai.privado.cache.{AppCache, DataFlowCache, RuleCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.model.exporter.ViolationDataFlowModel import ai.privado.model.PolicyOrThreat import ai.privado.policyEngine.PolicyExecutor @@ -50,13 +51,14 @@ object DataLeakageToNotifications { cpg: Cpg, dataflows: Map[String, Path], ruleCache: RuleCache, - dataFlowCache: DataFlowCache + dataFlowCache: DataFlowCache, + privadoInput: PrivadoInput ): Try[(Boolean, List[ViolationDataFlowModel])] = Try { // use policy executor to directly process existing flows (we have rule for notifications) // we already have this implementation as part of policy enforcement // threat being type of suggestive policy // might restructure this in future and have central utilities consumed by both - val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache) + val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, AppCache.repoName, ruleCache, privadoInput) val violatingFlows = policyExecutor.getViolatingFlowsForPolicy(threat) // violation if empty diff --git a/src/main/scala/ai/privado/languageEngine/java/threatEngine/ThreatEngineExecutor.scala b/src/main/scala/ai/privado/languageEngine/java/threatEngine/ThreatEngineExecutor.scala index 717c47a51..6f78f1980 100644 --- a/src/main/scala/ai/privado/languageEngine/java/threatEngine/ThreatEngineExecutor.scala +++ b/src/main/scala/ai/privado/languageEngine/java/threatEngine/ThreatEngineExecutor.scala @@ -24,6 +24,7 @@ package ai.privado.languageEngine.java.threatEngine import ai.privado.cache.{DataFlowCache, RuleCache, TaggerCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.exporter.ExporterUtility import ai.privado.model.exporter.ViolationModel import ai.privado.model.PolicyOrThreat @@ -40,7 +41,8 @@ class ThreatEngineExecutor( repoPath: String, ruleCache: RuleCache, taggerCache: TaggerCache, - dataFlowCache: DataFlowCache + dataFlowCache: DataFlowCache, + privadoInput: PrivadoInput ) { private val logger = LoggerFactory.getLogger(getClass) @@ -128,7 +130,7 @@ class ThreatEngineExecutor( } case "PrivadoPolicy.CookieConsent.IsCookieConsentMgmtModuleImplemented" => - CookieConsentMgmtModule.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache) match { + CookieConsentMgmtModule.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache, privadoInput) match { case Success(res) => Some(res) case Failure(e) => { logger.debug(s"Error for ${threatId}: ${e}") @@ -221,7 +223,7 @@ class ThreatEngineExecutor( val violationResponse = threatId match { case "Threats.Sharing.isDataExposedToThirdPartiesViaNotification" if isAndroidRepo => - DataLeakageToNotifications.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache) match { + DataLeakageToNotifications.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache, privadoInput) match { case Success(res) => Some(res) case Failure(e) => { logger.debug(s"Error for ${threatId}: ${e}") @@ -229,7 +231,7 @@ class ThreatEngineExecutor( } } case "Threats.Leakage.isDataLeakingToLog" => - DataLeakageToLogs.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache) match { + DataLeakageToLogs.getViolations(threat, cpg, dataflows, ruleCache, dataFlowCache, privadoInput) match { case Success(res) => Some(res) case Failure(e) => { logger.debug(s"Error for ${threatId}: ${e}") diff --git a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala index 9d878add1..e8290c81e 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala @@ -100,7 +100,8 @@ object JavascriptProcessor { dataflowMap, ruleCache, taggerCache, - dataFlowCache + dataFlowCache, + ScanProcessor.config ) match { case Left(err) => MetricHandler.otherErrorsOrWarnings.addOne(err) diff --git a/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala b/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala index c8aa77a80..29715fed5 100644 --- a/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala @@ -116,7 +116,8 @@ object PythonProcessor { dataflowMap, ruleCache, taggerCache, - dataFlowCache + dataFlowCache, + ScanProcessor.config ) match { case Left(err) => MetricHandler.otherErrorsOrWarnings.addOne(err) diff --git a/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala b/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala index a3ffd7003..a41f39a6a 100644 --- a/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala @@ -189,7 +189,8 @@ object RubyProcessor { dataflowMap, ruleCache, taggerCache, - dataFlowCache + dataFlowCache, + ScanProcessor.config ) match { case Left(err) => MetricHandler.otherErrorsOrWarnings.addOne(err) diff --git a/src/main/scala/ai/privado/policyEngine/PolicyExecutor.scala b/src/main/scala/ai/privado/policyEngine/PolicyExecutor.scala index 5c85c9119..59e0fe3c8 100644 --- a/src/main/scala/ai/privado/policyEngine/PolicyExecutor.scala +++ b/src/main/scala/ai/privado/policyEngine/PolicyExecutor.scala @@ -23,6 +23,7 @@ package ai.privado.policyEngine import ai.privado.cache.{DataFlowCache, RuleCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.exporter.ExporterUtility import ai.privado.languageEngine.java.threatEngine.ThreatUtility.getSourceNode import ai.privado.model.exporter.{ViolationDataFlowModel, ViolationProcessingModel} @@ -38,7 +39,13 @@ import overflowdb.traversal.Traversal import scala.collection.mutable import scala.util.{Failure, Success, Try} -class PolicyExecutor(cpg: Cpg, dataFlowCache: DataFlowCache, repoName: String, ruleCache: RuleCache) { +class PolicyExecutor( + cpg: Cpg, + dataFlowCache: DataFlowCache, + repoName: String, + ruleCache: RuleCache, + privadoInput: PrivadoInput +) { private val logger = LoggerFactory.getLogger(getClass) @@ -52,7 +59,7 @@ class PolicyExecutor(cpg: Cpg, dataFlowCache: DataFlowCache, repoName: String, r // Map to contain sinkId -> List(pathIds) lazy val dataflowSinkIdMap: Map[String, List[String]] = getDataflowBySinkIdMapping - val sourceExporter = new SourceExporter(cpg, ruleCache) + val sourceExporter = new SourceExporter(cpg, ruleCache, privadoInput) lazy val sourceExporterModel = sourceExporter.getSources diff --git a/src/test/scala/ai/privado/exporter/SourceExporterTest.scala b/src/test/scala/ai/privado/exporter/SourceExporterTest.scala new file mode 100644 index 000000000..d044856fa --- /dev/null +++ b/src/test/scala/ai/privado/exporter/SourceExporterTest.scala @@ -0,0 +1,46 @@ +package ai.privado.exporter + +import ai.privado.entrypoint.PrivadoInput +import ai.privado.languageEngine.java.JavaTaggingTestBase +import ai.privado.languageEngine.java.tagger.source.IdentifierTagger +import ai.privado.model.{CatLevelOne, Constants} +import io.shiftleft.semanticcpg.language.* +class SourceExporterTest extends JavaTaggingTestBase { + + override def beforeAll(): Unit = { + super.beforeAll() + new IdentifierTagger(cpg, ruleCache, taggerCache).createAndApply() + } + + override val javaFileContents = + """ + |class User { + | public String firstName; + | + | public String getFirstName() {return firstName;} + | public void setFirstName(String firstName) {this.firstName = firstName;} + |} + | + |class Auth { + | public display(User user) {System.out.println(user);} + |} + |""".stripMargin + + "Identifier Tagger" should { + "tag a derived source" in { + val identifierNodes = cpg.identifier("user").l + identifierNodes.size shouldBe 1 + identifierNodes.tag + .nameExact(Constants.catLevelOne) + .valueExact(CatLevelOne.DERIVED_SOURCES.name) + .nonEmpty shouldBe true + } + } + + "Source exporter" should { + "not export derived source under processing" in { + val sourceExporter = SourceExporter(cpg, ruleCache, PrivadoInput(disableDeDuplication = true)) + !sourceExporter.getProcessing.flatMap(_.occurrences).map(_.sample).exists(_.equals("user")) shouldBe true + } + } +} diff --git a/src/test/scala/ai/privado/languageEngine/java/JavaTaggingTestBase.scala b/src/test/scala/ai/privado/languageEngine/java/JavaTaggingTestBase.scala index a28dceddd..c740165b8 100644 --- a/src/test/scala/ai/privado/languageEngine/java/JavaTaggingTestBase.scala +++ b/src/test/scala/ai/privado/languageEngine/java/JavaTaggingTestBase.scala @@ -27,6 +27,7 @@ import ai.privado.cache.{AppCache, RuleCache, TaggerCache} import ai.privado.model.{CatLevelOne, ConfigAndRules, Language, NodeType, RuleInfo} import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} +import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers @@ -47,6 +48,7 @@ abstract class JavaTaggingTestBase extends AnyWordSpec with Matchers with Before outputFile = File.newTemporaryFile() val config = Config().withInputPath(inputDir.pathAsString).withOutputPath(outputFile.pathAsString) cpg = new JavaSrc2Cpg().createCpg(config).get + applyDefaultOverlays(cpg) // Caching Rule ruleCache.setRule(rule) diff --git a/src/test/scala/ai/privado/languageEngine/java/tagger/source/JavaIdentifierTaggingTest.scala b/src/test/scala/ai/privado/languageEngine/java/tagger/source/JavaIdentifierTaggingTest.scala index e0af06d06..81d18b3af 100644 --- a/src/test/scala/ai/privado/languageEngine/java/tagger/source/JavaIdentifierTaggingTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/tagger/source/JavaIdentifierTaggingTest.scala @@ -51,13 +51,11 @@ class JavaIdentifierTaggingTest extends JavaTaggingTestBase { identifierNodes.value.head shouldBe "Data.Sensitive.FirstName" } - /* "tag fieldAccess of firstName" in { // Note - this test is Fails with the current query on cpg.method.callIn, but works fine on cpg.call - val identifierNodes = cpg.call.tag.nameExact(Constants.id).l - identifierNodes.size shouldBe 1 + val identifierNodes = cpg.fieldAccess.tag.nameExact(Constants.id).l + identifierNodes.size shouldBe 2 identifierNodes.value.head shouldBe "Data.Sensitive.FirstName" } - */ } } diff --git a/src/test/scala/ai/privado/policyEngine/PolicyTests.scala b/src/test/scala/ai/privado/policyEngine/PolicyTests.scala index 58bca89b5..671a40791 100644 --- a/src/test/scala/ai/privado/policyEngine/PolicyTests.scala +++ b/src/test/scala/ai/privado/policyEngine/PolicyTests.scala @@ -135,7 +135,7 @@ class PolicyTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { new InSensitiveCallTagger(cpg, ruleCache, new TaggerCache()).createAndApply() new Dataflow(cpg).dataflow(privadoInput, ruleCache, dataFlowCache, auditCache) - val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, config.inputPath, ruleCache) + val policyExecutor = new PolicyExecutor(cpg, dataFlowCache, config.inputPath, ruleCache, privadoInput) policyExecutor } }