Release PR

schema/src/main/scala/CpgExtSchema.scala

@@ -147,7 +147,7 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) {
     .addProperty(artifactId)
     .addProperty(configVersion)
 
-  val dependency = builder
+  val moduleDependency = builder
     .addNodeType(CpgSchemaConstants.MODULE_DEPENDENCY_NODE_NAME)
     .addProperty(groupId)
     .addProperty(artifactId)
@@ -159,11 +159,17 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) {
   val dependencyModuleEdge = builder
     .addEdgeType(CpgSchemaConstants.DEPENDENCY_MODULE_EDGE_NAME)
 
-  module.addOutEdge(edge = dependencies, inNode = dependency)
+  module.addOutEdge(edge = dependencies, inNode = moduleDependency)
   module.addOutEdge(edge = sourceFile, inNode = file)
+  moduleDependency.addOutEdge(edge = sourceFile, inNode = file)
+  moduleDependency.addOutEdge(edge = dependencyModuleEdge, inNode = module)
+
+  // Extend Dependency Node
   dependency.addOutEdge(edge = sourceFile, inNode = file)
-  dependency.addOutEdge(edge = dependencyModuleEdge, inNode = module)
+  dependency.addOutEdge(edge = taggedBy, inNode = tag)
+  dependency.extendz(astNode)
 
+  // Extend teamplateDOM node
   templateDOM.addOutEdge(edge = sourceFile, inNode = file)
 
   // Nodes and edges for Android res/layout/*.xml files and
@@ -228,6 +234,7 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) {
   private val derivedSourceEdge  = builder.addEdgeType(CpgSchemaConstants.DERIVED_SOURCE_EDGE_NAME)
   astNode.addOutEdge(edge = originalSourceEdge, inNode = astNode)
   astNode.addOutEdge(edge = derivedSourceEdge, inNode = astNode)
+
 }
 
 object CpgExtSchema {

src/main/scala/ai/privado/entrypoint/CommandParser.scala

@@ -160,7 +160,7 @@ object CommandParser {
     Map(
       CommandConstants.SCAN     -> ScanProcessor,
       CommandConstants.UPLOAD   -> UploadProcessor,
-      CommandConstants.VALIDATE -> RuleValidator,
+      CommandConstants.VALIDATE -> RuleValidatorProcessor,
       CommandConstants.METADATA -> MetadataProcessor
     )
   def parse(args: Array[String], statsRecorder: StatsRecorder): Option[CommandProcessor] = {

src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala

@@ -1,25 +1,17 @@
 package ai.privado.entrypoint
 
-import ai.privado.cache.{
-  AppCache,
-  AuditCache,
-  DataFlowCache,
-  DatabaseDetailsCache,
-  FileLinkingMetadata,
-  PropertyFilterCache,
-  S3DatabaseDetailsCache
-}
+import ai.privado.cache.*
+import ai.privado.entrypoint.MetadataProcessor.statsRecorder
+import ai.privado.inputprocessor.RuleProcessor
 import ai.privado.languageEngine.javascript.processor.JavascriptBaseCPGProcessor
 import ai.privado.languageEngine.python.processor.PythonBaseCPGProcessor
 import ai.privado.metadata.SystemInfo
 import ai.privado.metric.MetricHandler
-import ai.privado.model.Constants
-import ai.privado.model.Language.UNKNOWN
 import ai.privado.model.*
+import ai.privado.model.Language.UNKNOWN
 import better.files.File
 import io.circe.Json
 import io.joern.console.cpgcreation.guessLanguage
-import ai.privado.entrypoint.MetadataProcessor.statsRecorder
 
 import scala.util.{Failure, Success, Try}
 

src/main/scala/ai/privado/entrypoint/RuleValidator.scala → src/main/scala/ai/privado/entrypoint/RuleValidatorProcessor.scala

@@ -6,7 +6,7 @@ import ai.privado.rulevalidator.YamlFileValidator
 import better.files.File
 import org.slf4j.LoggerFactory
 
-object RuleValidator extends CommandProcessor {
+object RuleValidatorProcessor extends CommandProcessor {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 

src/main/scala/ai/privado/entrypoint/ScanProcessor.scala

@@ -24,6 +24,7 @@ package ai.privado.entrypoint
 
 import ai.privado.cache.*
 import ai.privado.entrypoint.ScanProcessor.statsRecorder
+import ai.privado.inputprocessor.{DependencyInfo, DependencyTaggingProcessor, RuleProcessor}
 import ai.privado.languageEngine.c.processor.CProcessor
 import ai.privado.languageEngine.csharp.processor.CSharpProcessor
 import ai.privado.languageEngine.default.processor.DefaultProcessor
@@ -37,7 +38,7 @@ import ai.privado.languageEngine.ruby.processor.RubyProcessor
 import ai.privado.metadata.SystemInfo
 import ai.privado.metric.MetricHandler
 import ai.privado.model.*
-import ai.privado.model.Language.{Language, UNKNOWN}
+import ai.privado.model.Language.UNKNOWN
 import ai.privado.utility.StatsRecorder
 import better.files.File
 import io.circe.Json
@@ -47,9 +48,9 @@ import org.slf4j.LoggerFactory
 import privado_core.BuildInfo
 
 import scala.sys.exit
-import scala.util.{Try}
+import scala.util.Try
 
-object ScanProcessor extends CommandProcessor with RuleProcessor {
+object ScanProcessor extends CommandProcessor with RuleProcessor with DependencyTaggingProcessor {
   private val logger = LoggerFactory.getLogger(this.getClass)
 
   override def process(appCache: AppCache): Either[String, Unit] = {
@@ -96,7 +97,11 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
       config.forceLanguage
     }
     MetricHandler.metricsData("language") = Json.fromString(languageDetected.toString)
-
+    val dependencies: List[DependencyInfo] = config.externalConfigPath.headOption match {
+      case Some(externalConfigPath) =>
+        parseDependencyInfo(generateDependencyInfoJsonPath(externalConfigPath))
+      case None => List()
+    }
     languageDetected match {
       case Language.JAVA =>
         statsRecorder.justLogMessage("Detected language 'Java'")
@@ -118,7 +123,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
             statsRecorder = statsRecorder,
             databaseDetailsCache = databaseDetailsCache,
             propertyFilterCache = propertyFilterCache,
-            fileLinkingMetadata = fileLinkingMetadata
+            fileLinkingMetadata = fileLinkingMetadata,
+            dependencies = dependencies
           ).processCpg()
         else
           KotlinProcessor(
@@ -132,7 +138,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
             statsRecorder = statsRecorder,
             databaseDetailsCache = databaseDetailsCache,
             propertyFilterCache = propertyFilterCache,
-            fileLinkingMetadata = fileLinkingMetadata
+            fileLinkingMetadata = fileLinkingMetadata,
+            dependencies = dependencies
           ).processCpg()
       case Language.JAVASCRIPT =>
         statsRecorder.justLogMessage("Detected language 'JavaScript'")
@@ -147,7 +154,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
           statsRecorder = statsRecorder,
           databaseDetailsCache = databaseDetailsCache,
           propertyFilterCache = propertyFilterCache,
-          fileLinkingMetadata = fileLinkingMetadata
+          fileLinkingMetadata = fileLinkingMetadata,
+          dependencies = dependencies
         ).processCpg()
       case Language.PYTHON =>
         statsRecorder.justLogMessage("Detected language 'Python'")
@@ -162,7 +170,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
           propertyFilterCache = propertyFilterCache,
           databaseDetailsCache = databaseDetailsCache,
           statsRecorder = statsRecorder,
-          fileLinkingMetadata = fileLinkingMetadata
+          fileLinkingMetadata = fileLinkingMetadata,
+          dependencies = dependencies
         ).processCpg()
       case Language.RUBY =>
         statsRecorder.justLogMessage("Detected language 'Ruby'")
@@ -176,7 +185,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
           appCache,
           propertyFilterCache = propertyFilterCache,
           statsRecorder = statsRecorder,
-          fileLinkingMetadata = fileLinkingMetadata
+          fileLinkingMetadata = fileLinkingMetadata,
+          dependencies = dependencies
         ).processCpg()
       case Language.GO =>
         statsRecorder.justLogMessage("Detected language 'Go'")
@@ -191,7 +201,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
           propertyFilterCache = propertyFilterCache,
           statsRecorder = statsRecorder,
           databaseDetailsCache = databaseDetailsCache,
-          fileLinkingMetadata = fileLinkingMetadata
+          fileLinkingMetadata = fileLinkingMetadata,
+          dependencies = dependencies
         ).processCpg()
       case Language.KOTLIN =>
         statsRecorder.justLogMessage("Detected language 'Kotlin'")
@@ -206,7 +217,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
           statsRecorder = statsRecorder,
           databaseDetailsCache = databaseDetailsCache,
           propertyFilterCache = propertyFilterCache,
-          fileLinkingMetadata = fileLinkingMetadata
+          fileLinkingMetadata = fileLinkingMetadata,
+          dependencies = dependencies
         ).processCpg()
       case Language.CSHARP =>
         statsRecorder.justLogMessage("Detected language 'C#'")
@@ -221,7 +233,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
           statsRecorder = statsRecorder,
           databaseDetailsCache = databaseDetailsCache,
           propertyFilterCache = propertyFilterCache,
-          fileLinkingMetadata = fileLinkingMetadata
+          fileLinkingMetadata = fileLinkingMetadata,
+          dependencies = dependencies
         ).processCpg()
       case Language.PHP =>
         statsRecorder.justLogMessage("Detected language 'PHP'")
@@ -236,7 +249,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor {
           statsRecorder = statsRecorder,
           databaseDetailsCache = databaseDetailsCache,
           propertyFilterCache = propertyFilterCache,
-          fileLinkingMetadata = fileLinkingMetadata
+          fileLinkingMetadata = fileLinkingMetadata,
+          dependencies = dependencies
         )
           .processCpg()
       case Language.C =>

src/main/scala/ai/privado/exporter/SinkExporter.scala

@@ -145,7 +145,7 @@ class SinkExporter(
           .where(filterSink)
           .l ++ cpg.argument.isFieldIdentifier.where(filterSink).l ++ cpg.method.where(filterSink).l ++ cpg.dbNode
           .where(filterSink)
-          .l ++ cpg.highTouchSink.where(filterSink).l
+          .l ++ cpg.highTouchSink.where(filterSink).l ++ cpg.dependency.where(filterSink).l
     ExporterUtility.filterNodeBasedOnRepoItemTagName(sinks, repoItemTagName)
   }
 

src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala

@@ -0,0 +1,43 @@
+package ai.privado.inputprocessor
+
+import org.slf4j.LoggerFactory
+import upickle.default.*
+
+import java.io.File as JFile
+import java.nio.file.{Files, Paths}
+import scala.util.{Failure, Success, Try}
+case class DependencyInfo(
+  groupId: String,
+  dependencyName: String,
+  version: String,
+  code: String,
+  ruleId: String,
+  ruleName: String,
+  ruleDomains: List[String],
+  ruleTags: List[String],
+  lineNumber: Int,
+  filePath: String
+) {
+  def getFullDependencyName(): String = {
+    if groupId.isEmpty then dependencyName else s"$groupId.$dependencyName"
+  }
+}
+
+object DependencyInfo {
+  implicit val reader: Reader[DependencyInfo] = macroR[DependencyInfo]
+}
+trait DependencyTaggingProcessor {
+  private val logger = LoggerFactory.getLogger(this.getClass)
+
+  def generateDependencyInfoJsonPath(externalConfigPath: String): String =
+    List(externalConfigPath, "config", "dependencyInfo", "dependencyinfo.json").mkString(JFile.separator)
+  def parseDependencyInfo(filePath: String): List[DependencyInfo] = {
+    // Manually provide a reader for List[DependencyInfo]
+    Try(read[List[DependencyInfo]](new String(Files.readAllBytes(Paths.get(filePath))))) match {
+      case Success(dependencies) => dependencies
+      case Failure(exception) =>
+        logger.error(s"Error while parsing $filePath : ", exception)
+        List()
+    }
+  }
+}

src/main/scala/ai/privado/entrypoint/DynamicRuleMerger.scala → src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala

@@ -1,11 +1,10 @@
-package ai.privado.entrypoint
+package ai.privado.inputprocessor
 
 import ai.privado.model.{ConfigAndRules, FilterProperty, RuleInfo}
 import org.slf4j.LoggerFactory
 
 import scala.collection.mutable
-import scala.collection.mutable.Map
-import scala.collection.mutable.ListBuffer
+import scala.collection.mutable.{ListBuffer, Map}
 
 trait DynamicRuleMerger {
 

src/main/scala/ai/privado/entrypoint/RuleProcessor.scala → src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala

@@ -1,19 +1,20 @@
-package ai.privado.entrypoint
+package ai.privado.inputprocessor
 
 import ai.privado.cache.{AppCache, RuleCache}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.metric.MetricHandler
 import ai.privado.model.*
 import ai.privado.model.Language.Language
 import ai.privado.rulevalidator.YamlFileValidator
 import ai.privado.utility.StatsRecorder
+import ai.privado.utility.Utilities.{isValidDEDRule, isValidRule}
 import better.files.File
+import io.circe.Json
+import io.circe.yaml.parser
 import org.slf4j.LoggerFactory
 
 import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable
 import scala.sys.exit
-import io.circe.yaml.parser
-import ai.privado.utility.Utilities.{isValidDEDRule, isValidRule}
-import io.circe.Json
 
 trait RuleProcessor extends DynamicRuleMerger {
 

src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala

@@ -5,33 +5,31 @@ import ai.privado.cache.*
 import ai.privado.dataflow.Dataflow
 import ai.privado.entrypoint.PrivadoInput
 import ai.privado.exporter.{ExcelExporter, JSONExporter}
+import ai.privado.inputprocessor.DependencyInfo
 import ai.privado.languageEngine.java.cache.ModuleCache
-import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, ModuleFilePass}
+import ai.privado.languageEngine.java.passes.config.ModuleFilePass
 import ai.privado.languageEngine.java.passes.module.{DependenciesCategoryPass, DependenciesNodePass}
 import ai.privado.metric.MetricHandler
 import ai.privado.model.Constants.*
 import ai.privado.model.Language.Language
 import ai.privado.model.{CpgWithOutputMap, Language}
-import ai.privado.passes.ExperimentalLambdaDataFlowSupportPass
-import ai.privado.semantic.language.*
-import ai.privado.tagger.PrivadoParallelCpgPass
-import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility}
+import ai.privado.passes.{DependencyNodePass, ExperimentalLambdaDataFlowSupportPass}
+import ai.privado.tagger.sink.DependencyNodeTagger
+import ai.privado.utility.{StatsRecorder, UnresolvedReportUtility}
 import io.circe.Json
 import io.joern.dataflowengineoss.language.Path
 import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions}
-import io.joern.javasrc2cpg.Config
-import io.joern.x2cpg.X2CpgConfig
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.ModuleDependency
 import io.shiftleft.passes.CpgPassBase
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.layers.LayerCreatorContext
 import org.slf4j.{Logger, LoggerFactory}
 
-import java.util.Calendar
 import scala.collection.mutable
 import scala.collection.mutable.ListBuffer
 import scala.util.{Failure, Success, Try}
+
 abstract class BaseProcessor(
   ruleCache: RuleCache,
   privadoInput: PrivadoInput,
@@ -45,7 +43,8 @@ abstract class BaseProcessor(
   returnClosedCpg: Boolean,
   databaseDetailsCache: DatabaseDetailsCache,
   propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(),
-  fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata()
+  fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(),
+  dependencies: List[DependencyInfo] = List()
 ) {
 
   val logger: Logger = LoggerFactory.getLogger(getClass)
@@ -102,7 +101,7 @@ abstract class BaseProcessor(
     * @param cpg
     * @return
     */
-  def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List()
+  def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List(DependencyNodePass(cpg, dependencies, sourceRepoLocation))
 
   /** Method to apply Dataflow pass
     * @param cpg
@@ -154,7 +153,9 @@ abstract class BaseProcessor(
     result
   }
 
-  def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = ???
+  def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = {
+    DependencyNodeTagger(cpg, dependencies, ruleCache).createAndApply()
+  }
 
   protected def applyFinalExport(
     cpg: Cpg,