diff --git a/schema/src/main/scala/CpgExtSchema.scala b/schema/src/main/scala/CpgExtSchema.scala index 2550516b3..0807e1207 100644 --- a/schema/src/main/scala/CpgExtSchema.scala +++ b/schema/src/main/scala/CpgExtSchema.scala @@ -147,7 +147,7 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) { .addProperty(artifactId) .addProperty(configVersion) - val dependency = builder + val moduleDependency = builder .addNodeType(CpgSchemaConstants.MODULE_DEPENDENCY_NODE_NAME) .addProperty(groupId) .addProperty(artifactId) @@ -159,11 +159,17 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) { val dependencyModuleEdge = builder .addEdgeType(CpgSchemaConstants.DEPENDENCY_MODULE_EDGE_NAME) - module.addOutEdge(edge = dependencies, inNode = dependency) + module.addOutEdge(edge = dependencies, inNode = moduleDependency) module.addOutEdge(edge = sourceFile, inNode = file) + moduleDependency.addOutEdge(edge = sourceFile, inNode = file) + moduleDependency.addOutEdge(edge = dependencyModuleEdge, inNode = module) + + // Extend Dependency Node dependency.addOutEdge(edge = sourceFile, inNode = file) - dependency.addOutEdge(edge = dependencyModuleEdge, inNode = module) + dependency.addOutEdge(edge = taggedBy, inNode = tag) + dependency.extendz(astNode) + // Extend teamplateDOM node templateDOM.addOutEdge(edge = sourceFile, inNode = file) // Nodes and edges for Android res/layout/*.xml files and @@ -228,6 +234,7 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) { private val derivedSourceEdge = builder.addEdgeType(CpgSchemaConstants.DERIVED_SOURCE_EDGE_NAME) astNode.addOutEdge(edge = originalSourceEdge, inNode = astNode) astNode.addOutEdge(edge = derivedSourceEdge, inNode = astNode) + } object CpgExtSchema { diff --git a/src/main/scala/ai/privado/entrypoint/CommandParser.scala b/src/main/scala/ai/privado/entrypoint/CommandParser.scala index db7d4c76a..0cebbf3c3 100644 --- a/src/main/scala/ai/privado/entrypoint/CommandParser.scala +++ b/src/main/scala/ai/privado/entrypoint/CommandParser.scala @@ -160,7 +160,7 @@ object CommandParser { Map( CommandConstants.SCAN -> ScanProcessor, CommandConstants.UPLOAD -> UploadProcessor, - CommandConstants.VALIDATE -> RuleValidator, + CommandConstants.VALIDATE -> RuleValidatorProcessor, CommandConstants.METADATA -> MetadataProcessor ) def parse(args: Array[String], statsRecorder: StatsRecorder): Option[CommandProcessor] = { diff --git a/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala b/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala index 9faaa260f..a6b1c2ead 100644 --- a/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala +++ b/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala @@ -1,25 +1,17 @@ package ai.privado.entrypoint -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - FileLinkingMetadata, - PropertyFilterCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* +import ai.privado.entrypoint.MetadataProcessor.statsRecorder +import ai.privado.inputprocessor.RuleProcessor import ai.privado.languageEngine.javascript.processor.JavascriptBaseCPGProcessor import ai.privado.languageEngine.python.processor.PythonBaseCPGProcessor import ai.privado.metadata.SystemInfo import ai.privado.metric.MetricHandler -import ai.privado.model.Constants -import ai.privado.model.Language.UNKNOWN import ai.privado.model.* +import ai.privado.model.Language.UNKNOWN import better.files.File import io.circe.Json import io.joern.console.cpgcreation.guessLanguage -import ai.privado.entrypoint.MetadataProcessor.statsRecorder import scala.util.{Failure, Success, Try} diff --git a/src/main/scala/ai/privado/entrypoint/RuleValidator.scala b/src/main/scala/ai/privado/entrypoint/RuleValidatorProcessor.scala similarity index 98% rename from src/main/scala/ai/privado/entrypoint/RuleValidator.scala rename to src/main/scala/ai/privado/entrypoint/RuleValidatorProcessor.scala index f997233ef..78834b8f3 100644 --- a/src/main/scala/ai/privado/entrypoint/RuleValidator.scala +++ b/src/main/scala/ai/privado/entrypoint/RuleValidatorProcessor.scala @@ -6,7 +6,7 @@ import ai.privado.rulevalidator.YamlFileValidator import better.files.File import org.slf4j.LoggerFactory -object RuleValidator extends CommandProcessor { +object RuleValidatorProcessor extends CommandProcessor { private val logger = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala b/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala index 99fcdfe4d..8b05a36cb 100644 --- a/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala +++ b/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala @@ -24,6 +24,7 @@ package ai.privado.entrypoint import ai.privado.cache.* import ai.privado.entrypoint.ScanProcessor.statsRecorder +import ai.privado.inputprocessor.{DependencyInfo, DependencyTaggingProcessor, RuleProcessor} import ai.privado.languageEngine.c.processor.CProcessor import ai.privado.languageEngine.csharp.processor.CSharpProcessor import ai.privado.languageEngine.default.processor.DefaultProcessor @@ -37,7 +38,7 @@ import ai.privado.languageEngine.ruby.processor.RubyProcessor import ai.privado.metadata.SystemInfo import ai.privado.metric.MetricHandler import ai.privado.model.* -import ai.privado.model.Language.{Language, UNKNOWN} +import ai.privado.model.Language.UNKNOWN import ai.privado.utility.StatsRecorder import better.files.File import io.circe.Json @@ -47,9 +48,9 @@ import org.slf4j.LoggerFactory import privado_core.BuildInfo import scala.sys.exit -import scala.util.{Try} +import scala.util.Try -object ScanProcessor extends CommandProcessor with RuleProcessor { +object ScanProcessor extends CommandProcessor with RuleProcessor with DependencyTaggingProcessor { private val logger = LoggerFactory.getLogger(this.getClass) override def process(appCache: AppCache): Either[String, Unit] = { @@ -96,7 +97,11 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { config.forceLanguage } MetricHandler.metricsData("language") = Json.fromString(languageDetected.toString) - + val dependencies: List[DependencyInfo] = config.externalConfigPath.headOption match { + case Some(externalConfigPath) => + parseDependencyInfo(generateDependencyInfoJsonPath(externalConfigPath)) + case None => List() + } languageDetected match { case Language.JAVA => statsRecorder.justLogMessage("Detected language 'Java'") @@ -118,7 +123,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() else KotlinProcessor( @@ -132,7 +138,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.JAVASCRIPT => statsRecorder.justLogMessage("Detected language 'JavaScript'") @@ -147,7 +154,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.PYTHON => statsRecorder.justLogMessage("Detected language 'Python'") @@ -162,7 +170,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { propertyFilterCache = propertyFilterCache, databaseDetailsCache = databaseDetailsCache, statsRecorder = statsRecorder, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.RUBY => statsRecorder.justLogMessage("Detected language 'Ruby'") @@ -176,7 +185,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { appCache, propertyFilterCache = propertyFilterCache, statsRecorder = statsRecorder, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.GO => statsRecorder.justLogMessage("Detected language 'Go'") @@ -191,7 +201,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { propertyFilterCache = propertyFilterCache, statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.KOTLIN => statsRecorder.justLogMessage("Detected language 'Kotlin'") @@ -206,7 +217,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.CSHARP => statsRecorder.justLogMessage("Detected language 'C#'") @@ -221,7 +233,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.PHP => statsRecorder.justLogMessage("Detected language 'PHP'") @@ -236,7 +249,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) .processCpg() case Language.C => diff --git a/src/main/scala/ai/privado/exporter/SinkExporter.scala b/src/main/scala/ai/privado/exporter/SinkExporter.scala index 7b8e9e53c..f93c45bdd 100644 --- a/src/main/scala/ai/privado/exporter/SinkExporter.scala +++ b/src/main/scala/ai/privado/exporter/SinkExporter.scala @@ -145,7 +145,7 @@ class SinkExporter( .where(filterSink) .l ++ cpg.argument.isFieldIdentifier.where(filterSink).l ++ cpg.method.where(filterSink).l ++ cpg.dbNode .where(filterSink) - .l ++ cpg.highTouchSink.where(filterSink).l + .l ++ cpg.highTouchSink.where(filterSink).l ++ cpg.dependency.where(filterSink).l ExporterUtility.filterNodeBasedOnRepoItemTagName(sinks, repoItemTagName) } diff --git a/src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala b/src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala new file mode 100644 index 000000000..3dac778e4 --- /dev/null +++ b/src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala @@ -0,0 +1,43 @@ +package ai.privado.inputprocessor + +import org.slf4j.LoggerFactory +import upickle.default.* + +import java.io.File as JFile +import java.nio.file.{Files, Paths} +import scala.util.{Failure, Success, Try} +case class DependencyInfo( + groupId: String, + dependencyName: String, + version: String, + code: String, + ruleId: String, + ruleName: String, + ruleDomains: List[String], + ruleTags: List[String], + lineNumber: Int, + filePath: String +) { + def getFullDependencyName(): String = { + if groupId.isEmpty then dependencyName else s"$groupId.$dependencyName" + } +} + +object DependencyInfo { + implicit val reader: Reader[DependencyInfo] = macroR[DependencyInfo] +} +trait DependencyTaggingProcessor { + private val logger = LoggerFactory.getLogger(this.getClass) + + def generateDependencyInfoJsonPath(externalConfigPath: String): String = + List(externalConfigPath, "config", "dependencyInfo", "dependencyinfo.json").mkString(JFile.separator) + def parseDependencyInfo(filePath: String): List[DependencyInfo] = { + // Manually provide a reader for List[DependencyInfo] + Try(read[List[DependencyInfo]](new String(Files.readAllBytes(Paths.get(filePath))))) match { + case Success(dependencies) => dependencies + case Failure(exception) => + logger.error(s"Error while parsing $filePath : ", exception) + List() + } + } +} diff --git a/src/main/scala/ai/privado/entrypoint/DynamicRuleMerger.scala b/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala similarity index 94% rename from src/main/scala/ai/privado/entrypoint/DynamicRuleMerger.scala rename to src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala index 63a8fd0f4..3dfdfb14e 100644 --- a/src/main/scala/ai/privado/entrypoint/DynamicRuleMerger.scala +++ b/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala @@ -1,11 +1,10 @@ -package ai.privado.entrypoint +package ai.privado.inputprocessor import ai.privado.model.{ConfigAndRules, FilterProperty, RuleInfo} import org.slf4j.LoggerFactory import scala.collection.mutable -import scala.collection.mutable.Map -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{ListBuffer, Map} trait DynamicRuleMerger { diff --git a/src/main/scala/ai/privado/entrypoint/RuleProcessor.scala b/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala similarity index 99% rename from src/main/scala/ai/privado/entrypoint/RuleProcessor.scala rename to src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala index b89cc68a2..54fb69ece 100644 --- a/src/main/scala/ai/privado/entrypoint/RuleProcessor.scala +++ b/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala @@ -1,19 +1,20 @@ -package ai.privado.entrypoint +package ai.privado.inputprocessor import ai.privado.cache.{AppCache, RuleCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.metric.MetricHandler import ai.privado.model.* import ai.privado.model.Language.Language import ai.privado.rulevalidator.YamlFileValidator import ai.privado.utility.StatsRecorder +import ai.privado.utility.Utilities.{isValidDEDRule, isValidRule} import better.files.File +import io.circe.Json +import io.circe.yaml.parser import org.slf4j.LoggerFactory import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable import scala.sys.exit -import io.circe.yaml.parser -import ai.privado.utility.Utilities.{isValidDEDRule, isValidRule} -import io.circe.Json trait RuleProcessor extends DynamicRuleMerger { diff --git a/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala b/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala index 07e33dd4c..e250ce8a8 100644 --- a/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala @@ -5,22 +5,20 @@ import ai.privado.cache.* import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.PrivadoInput import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.java.cache.ModuleCache -import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, ModuleFilePass} +import ai.privado.languageEngine.java.passes.config.ModuleFilePass import ai.privado.languageEngine.java.passes.module.{DependenciesCategoryPass, DependenciesNodePass} import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* import ai.privado.model.Language.Language import ai.privado.model.{CpgWithOutputMap, Language} -import ai.privado.passes.ExperimentalLambdaDataFlowSupportPass -import ai.privado.semantic.language.* -import ai.privado.tagger.PrivadoParallelCpgPass -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} +import ai.privado.passes.{DependencyNodePass, ExperimentalLambdaDataFlowSupportPass} +import ai.privado.tagger.sink.DependencyNodeTagger +import ai.privado.utility.{StatsRecorder, UnresolvedReportUtility} import io.circe.Json import io.joern.dataflowengineoss.language.Path import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} -import io.joern.javasrc2cpg.Config -import io.joern.x2cpg.X2CpgConfig import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.ModuleDependency import io.shiftleft.passes.CpgPassBase @@ -28,10 +26,10 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.{Logger, LoggerFactory} -import java.util.Calendar import scala.collection.mutable import scala.collection.mutable.ListBuffer import scala.util.{Failure, Success, Try} + abstract class BaseProcessor( ruleCache: RuleCache, privadoInput: PrivadoInput, @@ -45,7 +43,8 @@ abstract class BaseProcessor( returnClosedCpg: Boolean, databaseDetailsCache: DatabaseDetailsCache, propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] = List() ) { val logger: Logger = LoggerFactory.getLogger(getClass) @@ -102,7 +101,7 @@ abstract class BaseProcessor( * @param cpg * @return */ - def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List() + def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List(DependencyNodePass(cpg, dependencies, sourceRepoLocation)) /** Method to apply Dataflow pass * @param cpg @@ -154,7 +153,9 @@ abstract class BaseProcessor( result } - def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = ??? + def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + DependencyNodeTagger(cpg, dependencies, ruleCache).createAndApply() + } protected def applyFinalExport( cpg: Cpg, diff --git a/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala b/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala index 0a3279ce4..a328528e1 100644 --- a/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala @@ -23,40 +23,24 @@ package ai.privado.languageEngine.csharp.processor -import ai.privado.audit.AuditReportEntryPoint import ai.privado.cache.* -import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.* import ai.privado.entrypoint.ScanProcessor.config -import ai.privado.entrypoint.ScanProcessor -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.csharp.semantic.Language.tagger -import ai.privado.languageEngine.java.passes.config.JavaPropertyLinkerPass -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* -import ai.privado.model.Language.Language -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} -import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.model.{Constants, CpgWithOutputMap, Language} +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder} -import better.files.File -import io.circe.Json import io.joern.csharpsrc2cpg.* -import io.joern.dataflowengineoss.language.Path -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.x2cpg.X2Cpg import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.LoggerFactory -import java.util.Calendar -import scala.collection.mutable.ListBuffer -import scala.util.{Failure, Success, Try} +import scala.util.Try class CSharpProcessor( ruleCache: RuleCache, @@ -70,7 +54,8 @@ class CSharpProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -84,15 +69,17 @@ class CSharpProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { private val logger = LoggerFactory.getLogger(getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List[CpgPassBase]() + super.applyPrivadoPasses(cpg) } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala b/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala index 5785df6d1..888c4973d 100644 --- a/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala @@ -1,29 +1,25 @@ package ai.privado.languageEngine.go.processor -import ai.privado.audit.AuditReportEntryPoint import ai.privado.cache.* -import ai.privado.dataflow.Dataflow -import ai.privado.entrypoint.ScanProcessor.config import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.go.passes.SQLQueryParser import ai.privado.languageEngine.go.passes.config.GoYamlLinkerPass import ai.privado.languageEngine.go.passes.orm.ORMParserPass import ai.privado.languageEngine.go.semantic.Language.tagger -import ai.privado.semantic.* import ai.privado.model.Constants.* -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.semantic.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder} import io.joern.gosrc2cpg.{Config, GoSrc2Cpg} import io.joern.x2cpg.X2Cpg import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory class GoProcessor( @@ -38,7 +34,8 @@ class GoProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -52,12 +49,13 @@ class GoProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { private val logger = LoggerFactory.getLogger(getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List( + super.applyPrivadoPasses(cpg) ++ List( { if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") @@ -72,6 +70,7 @@ class GoProcessor( } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala index dea731054..2e4895aa1 100644 --- a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala @@ -23,42 +23,30 @@ package ai.privado.languageEngine.java.processor -import ai.privado.audit.{AuditReportEntryPoint, DependencyReport} import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor -import ai.privado.languageEngine.java.cache.ModuleCache -import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, JavaYamlLinkerPass, ModuleFilePass} +import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, JavaYamlLinkerPass} import ai.privado.languageEngine.java.passes.methodFullName.LoggerLombokPass -import ai.privado.languageEngine.java.passes.module.{DependenciesCategoryPass, DependenciesNodePass} import ai.privado.languageEngine.java.semantic.Language.* -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* -import ai.privado.model.Language.Language -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* import ai.privado.tagger.PrivadoParallelCpgPass +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} import better.files.File -import io.circe.Json -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.joern.x2cpg.utils.ExternalCommand import io.joern.x2cpg.utils.dependency.DependencyResolver +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.JavaProperty -import io.shiftleft.codepropertygraph.generated.{Cpg, Languages} import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.{Logger, LoggerFactory} import java.nio.file.Paths -import java.util.Calendar -import scala.collection.mutable.ListBuffer import scala.util.{Failure, Success, Try} class JavaProcessor( @@ -73,7 +61,8 @@ class JavaProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -87,14 +76,15 @@ class JavaProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger: Logger = LoggerFactory.getLogger(getClass) private var cpgconfig = Config() override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List({ + super.applyPrivadoPasses(cpg) ++ List({ if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") else @@ -110,7 +100,8 @@ class JavaProcessor( ) } - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -122,6 +113,7 @@ class JavaProcessor( statsRecorder, fileLinkingMetadata ) + } override def processCpg(): Either[String, CpgWithOutputMap] = { val excludeFileRegex = ruleCache.getExclusionRegex diff --git a/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala b/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala index ea6684dbc..94ac4f601 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala @@ -1,19 +1,44 @@ package ai.privado.languageEngine.javascript.metadata -import ai.privado.cache.FileLinkingMetadata +import ai.privado.cache.{AppCache, FileLinkingMetadata, RuleCache} +import ai.privado.passes.JsonParser import io.joern.x2cpg.passes.frontend.XImportResolverPass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call import java.util.regex.{Matcher, Pattern} -import java.io.File as JFile -import scala.util.Try +import java.io.{FileNotFoundException, File as JFile} +import better.files.File +import better.files.File.VisitOptions +import io.joern.x2cpg.SourceFiles -class FileImportMappingPassJS(cpg: Cpg, fileLinkingMetadata: FileLinkingMetadata) - extends XImportResolverPass(cpg: Cpg) { +import java.util.concurrent.ConcurrentHashMap +import scala.collection.mutable +import scala.util.control.Breaks.{break, breakable} +import scala.util.{Failure, Success, Try} + +class FileImportMappingPassJS( + cpg: Cpg, + fileLinkingMetadata: FileLinkingMetadata, + appCache: AppCache, + ruleCache: RuleCache +) extends XImportResolverPass(cpg: Cpg) + with JsonParser { private val pathPattern = Pattern.compile("[\"']([\\w/.]+)[\"']") + val sep: String = Matcher.quoteReplacement(JFile.separator) + val root = s"${sanitiseProbeScanPath(codeRootDir)}${JFile.separator}" + + private val tsConfigPathMapping = mutable.HashMap[String, String]() + + private val tsConfigEntityMissCache = ConcurrentHashMap.newKeySet[String]() + + override def init(): Unit = { + // initialize tsconfig.json map + initializeConfigMap() + } + override protected def optionalResolveImport( fileName: String, importCall: Call, @@ -21,46 +46,155 @@ class FileImportMappingPassJS(cpg: Cpg, fileLinkingMetadata: FileLinkingMetadata importedAs: String, diffGraph: DiffGraphBuilder ): Unit = { - val pathSep = ":" - val rawEntity = importedEntity.stripPrefix("./") - val alias = importedAs - val matcher = pathPattern.matcher(rawEntity) - val sep = Matcher.quoteReplacement(JFile.separator) - val root = s"$codeRootDir${JFile.separator}" - val currentFile = s"$root$fileName" - val extension = better.files.File(currentFile).`extension`.getOrElse(".ts") + val pathSep = ":" + val currentFile = s"$root$fileName" + val extension = File(currentFile).`extension`.getOrElse(".ts") + val parentDirPath = File(currentFile).parent.pathAsString // Stores the path of the parent dir of current file + + val importedModule = getImportingModule(importedEntity, pathSep) + // We want to know if the import is local since if an external name is used to match internal methods we may have // false paths. - val isLocalImport = importedEntity.matches("^[.]+/?.*") - // TODO: At times there is an operation inside of a require, e.g. path.resolve(__dirname + "/../config/env/all.js") - // this tries to recover the string but does not perform string constant propagation - val entity = if (matcher.find()) matcher.group(1) else rawEntity - - val isImportingModule = !entity.contains(pathSep) - if (isLocalImport) { - val resolvedPath = Try( - better.files - .File(currentFile.stripSuffix(currentFile.split(sep).last), entity.split(pathSep).head) - .pathAsString - .stripPrefix(root) - ).getOrElse(entity) - fileLinkingMetadata.addToFileImportMap(fileName, s"$resolvedPath$extension") - } else { - val seperatedFilePathList = fileName.split(sep).toList - val startingModule = entity.split(sep).head - val moduleIndex = seperatedFilePathList.indexOf(startingModule) - if (moduleIndex != -1) { - Try { - val resolvedPath = better.files - .File(root, seperatedFilePathList.take(moduleIndex).mkString(sep), entity.split(pathSep).head) - .pathAsString - .stripPrefix(root) - fileLinkingMetadata.addToFileImportMap(fileName, s"$resolvedPath$extension") + val isRelativeImport = importedEntity.matches("^[.]+/?.*") + + if (isRelativeImport && importedModule.isDefined) { + getResolvedPath(parentDirPath, importedModule.get, importedAs, extension) match + case Failure(_) => // unable to resolve + case Success(resolvedPath) => fileLinkingMetadata.addToFileImportMap(fileName, resolvedPath) + } else if (importedModule.isDefined) { + val relativeDirCount = parentDirPath.stripPrefix(root).split(sep).length + breakable { + for (i <- 0 to relativeDirCount) { + val resolvedPath = + getResolvedPath( + parentDirPath.split(sep).dropRight(i).mkString(sep), + importedModule.get, + importedAs, + extension + ) + if (resolvedPath.isSuccess) { + fileLinkingMetadata.addToFileImportMap(fileName, resolvedPath.get) + break + } } } + } + } + + /** Function to get us a a probable relative file path, if exists for the importing module based on input parameters + * @param parentDirPath + * \- parent dir path where we intend to do the lookup + * @param relativePath + * \- importing module path + * @param importedAs + * \- importedAs value of import + * @param currentFileExtension + * \- current extension of the file being processed + * @return + */ + def getResolvedPath( + parentDirPath: String, + relativePath: String, + importedAs: String, + currentFileExtension: String + ): Try[String] = + Try { + val file = File(parentDirPath, relativePath) + if (file.exists && file.isRegularFile) { + file.pathAsString.stripPrefix(root) + } else { + // If not found, try to find a file with the same name extension + val baseName = file.nameWithoutExtension + val parentDir = file.parent + val fileWithSameNames = parentDir.list.filter { f => + f.isRegularFile && f.nameWithoutExtension == baseName + }.toList + + // If multiple files match with sameName, prefer the one having same extension + fileWithSameNames.size match + case size if size == 0 => throw FileNotFoundException() + case size if size == 1 => fileWithSameNames.head.pathAsString.stripPrefix(root) + case _ => + fileWithSameNames.find(f => f.`extension`.exists(_.equals(currentFileExtension))) match + case Some(fileWithSameNameAndExtension) => fileWithSameNameAndExtension.pathAsString.stripPrefix(root) + case None => fileWithSameNames.head.pathAsString.stripPrefix(root) + } + } + + /** From ImportedEntity after applying some lookup gives the importing Module + * @param importedEntity + * \- The value present as part of import or require statement + * @param pathSep + * \- The path sep used to concat the importing modules elements + * @return + */ + private def getImportingModule(importedEntity: String, pathSep: String) = { + importedEntity.split(pathSep).head match + case entity if entity.startsWith("@") => + // if import starts with `@` this can mean import of local modules in some case + if (tsConfigPathMapping.contains(entity)) { + Some(tsConfigPathMapping(entity)) + } else if (tsConfigEntityMissCache.contains(entity)) + None + else { + tsConfigPathMapping.keys.filter(_.endsWith("*")).find { k => + val keyRegex = k.replace("*", ".*").r + val value = keyRegex.matches(entity) + value + } match + case Some(configKey) => + val configPathValue = tsConfigPathMapping(configKey).stripSuffix("*") + val resolvedModule = entity.replace(configKey.stripSuffix("*"), configPathValue) + // println(s"ResolvedModule : $resolvedModule, for $entity and $importedEntity") + Some(resolvedModule) + case None => + tsConfigEntityMissCache.add(entity) + None + } + case entity => Some(entity) + } + + /** Returns all the file paths where tsconfig.json or jsconfig.json are defined + * @return + */ + private def getJsonPathConfigFiles: List[String] = { + val repoPath = sanitiseProbeScanPath(appCache.scanPath) + val filePaths = SourceFiles + .determine(repoPath, Set(".json"), ignoredFilesRegex = Option(ruleCache.getExclusionRegex.r))( + VisitOptions.default + ) + val filteredFilePaths = filePaths.filter { fp => + val f = File(fp) + f.nameWithoutExtension.contains("tsconfig") || f.nameWithoutExtension.contains("jsconfig") } + filteredFilePaths + } + /** Initializes the configuration map by reading the configurations files + */ + private def initializeConfigMap(): Unit = { + val compilerPathConstant = "compilerOptions.paths" + val configFilePaths = getJsonPathConfigFiles + + configFilePaths.foreach { configFilePath => + getJSONKeyValuePairs(configFilePath) + .filter(_._1.contains(compilerPathConstant)) + .foreach { pathEntry => + // do clean up of the paths key + // We would get keys like - compilerOptions.paths.@utils/*[0] + val pathKey = pathEntry._1.split(s"${compilerPathConstant}.").last.split("\\[").head + val pathValue = pathEntry._2 + tsConfigPathMapping.addOne(pathKey, pathValue) + } + } } + /** Function to sanitise the scanPath and remove `/probe` as we copy files in /probe folder, this helps in lookup for + * resolvedPaths + * @param scanPath + * \- repo path used for scanning + * @return + */ + private def sanitiseProbeScanPath(scanPath: String) = scanPath.replace(s"${sep}probe", "") } diff --git a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala index 0a5cd342e..503d3dc65 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala @@ -12,6 +12,7 @@ import ai.privado.cache.{ TaggerCache } import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.metric.MetricHandler import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} import ai.privado.model.CpgWithOutputMap @@ -49,7 +50,8 @@ class JavascriptBaseCPGProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + List() ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala index 6735357cb..3aadb8158 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala @@ -23,38 +23,25 @@ package ai.privado.languageEngine.javascript.processor -import ai.privado.audit.AuditReportEntryPoint import ai.privado.cache.* -import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.PrivadoInput -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.javascript.metadata.FileImportMappingPassJS import ai.privado.languageEngine.javascript.passes.config.{JSPropertyLinkerPass, JsConfigPropertyPass} import ai.privado.languageEngine.javascript.semantic.Language.* -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} -import better.files.File import io.joern.jssrc2cpg.{Config, JsSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays -import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.shiftleft.codepropertygraph -import io.shiftleft.codepropertygraph.generated.{Cpg, Operators} +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import java.nio.file.Paths -import java.util.Calendar -import scala.collection.mutable.ListBuffer -import scala.jdk.CollectionConverters.CollectionHasAsScala -import scala.util.{Failure, Success, Try} - class JavascriptProcessor( ruleCache: RuleCache, privadoInput: PrivadoInput, @@ -67,7 +54,8 @@ class JavascriptProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -81,15 +69,16 @@ class JavascriptProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - val passesList = List(new HTMLParserPass(cpg, sourceRepoLocation, ruleCache, privadoInputConfig = privadoInput)) - - passesList ++ List({ + super.applyPrivadoPasses(cpg) ++ List( + new HTMLParserPass(cpg, sourceRepoLocation, ruleCache, privadoInputConfig = privadoInput) + ) ++ List({ if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") new JsConfigPropertyPass(cpg) @@ -110,7 +99,8 @@ class JavascriptProcessor( ) } - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -121,7 +111,7 @@ class JavascriptProcessor( statsRecorder, fileLinkingMetadata ) - + } override def applyDataflowAndPostProcessingPasses(cpg: Cpg): Unit = { super.applyDataflowAndPostProcessingPasses(cpg) if (privadoInput.disablePostProcessingPass) @@ -129,7 +119,7 @@ class JavascriptProcessor( else JsSrc2Cpg.postProcessingPasses(cpg).foreach(_.createAndApply()) if (privadoInput.fileLinkingReport) { - new FileImportMappingPassJS(cpg, fileLinkingMetadata).createAndApply() + new FileImportMappingPassJS(cpg, fileLinkingMetadata, appCache, ruleCache).createAndApply() } } diff --git a/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala b/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala index e0bd2f0ad..708fb1850 100644 --- a/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala @@ -1,50 +1,22 @@ package ai.privado.languageEngine.kotlin.processor -import ai.privado.audit.{AuditReportEntryPoint, DependencyReport} -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - FileLinkingMetadata, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache, - TaggerCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor -import ai.privado.languageEngine.java.cache.ModuleCache -import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, ModuleFilePass} -import ai.privado.languageEngine.java.passes.module.{DependenciesCategoryPass, DependenciesNodePass} +import ai.privado.languageEngine.java.passes.config.JavaPropertyLinkerPass import ai.privado.languageEngine.kotlin.semantic.Language.* -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* -import ai.privado.model.Language.Language -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} -import better.files.File -import io.circe.Json -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.kotlin2cpg.{Config, Kotlin2Cpg} import io.joern.x2cpg.X2Cpg -import io.joern.x2cpg.passes.base.AstLinkerPass -import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.LoggerFactory - -import java.nio.file.Paths -import java.util.Calendar -import scala.collection.mutable.ListBuffer -import scala.util.{Failure, Success, Try} class KotlinProcessor( ruleCache: RuleCache, privadoInput: PrivadoInput, @@ -57,7 +29,8 @@ class KotlinProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -71,13 +44,14 @@ class KotlinProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger = LoggerFactory.getLogger(getClass) private var cpgconfig = Config() override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List({ + super.applyPrivadoPasses(cpg) ++ List({ if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") else @@ -99,7 +73,8 @@ class KotlinProcessor( statsRecorder.endLastStage() } - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -110,6 +85,7 @@ class KotlinProcessor( statsRecorder, fileLinkingMetadata ) + } override def processCpg(): Either[String, CpgWithOutputMap] = { diff --git a/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala b/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala index 35b27be06..49d08fb4e 100644 --- a/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala @@ -24,19 +24,16 @@ package ai.privado.languageEngine.php.processor import ai.privado.cache.* -import ai.privado.entrypoint.ScanProcessor.config import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.php.semantic.Language.tagger import ai.privado.model.Constants.* import ai.privado.model.{CpgWithOutputMap, Language} -import ai.privado.model.Language.Language -import ai.privado.model.{CpgWithOutputMap, Language} import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import io.circe.Json import io.joern.php2cpg.{Config, Php2Cpg} -import io.joern.x2cpg.X2Cpg.{applyDefaultOverlays, newEmptyCpg} +import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase import org.slf4j.{Logger, LoggerFactory} @@ -44,7 +41,6 @@ import org.slf4j.{Logger, LoggerFactory} import java.io.File import java.nio.file.Paths import java.util.Calendar -import scala.util.Try class PhpProcessor( ruleCache: RuleCache, @@ -58,7 +54,8 @@ class PhpProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -72,14 +69,16 @@ class PhpProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) - override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List[CpgPassBase]() + override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = super.applyPrivadoPasses(cpg) - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -90,6 +89,7 @@ class PhpProcessor( statsRecorder, fileLinkingMetadata ) + } override def applyDataflowAndPostProcessingPasses(cpg: Cpg): Unit = { super.applyDataflowAndPostProcessingPasses(cpg) diff --git a/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala b/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala index 10a68102d..a659b689e 100644 --- a/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala @@ -1,16 +1,6 @@ package ai.privado.languageEngine.python.processor -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - FileLinkingMetadata, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache, - TaggerCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput import ai.privado.metric.MetricHandler import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} @@ -49,7 +39,8 @@ class PythonBaseCPGProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + List() ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala b/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala index 880fbd7be..ff8b85c9e 100644 --- a/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala @@ -1,20 +1,19 @@ package ai.privado.languageEngine.python.processor -import ai.privado.entrypoint.PrivadoInput import ai.privado.cache.* +import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.python.config.PythonConfigPropertyPass import ai.privado.languageEngine.python.metadata.FileLinkingMetadataPassPython import ai.privado.languageEngine.python.passes.PrivadoPythonTypeHintCallLinker import ai.privado.languageEngine.python.passes.config.PythonPropertyLinkerPass import ai.privado.languageEngine.python.semantic.Language.* -import ai.privado.languageEngine.python.tagger.PythonS3Tagger import ai.privado.model.Constants.* import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder} import better.files.File import io.joern.pysrc2cpg.* import io.joern.x2cpg.X2Cpg.applyDefaultOverlays @@ -23,7 +22,6 @@ import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory import java.nio.file.Paths @@ -40,7 +38,8 @@ class PythonProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -54,13 +53,14 @@ class PythonProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger = LoggerFactory.getLogger(getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List( + super.applyPrivadoPasses(cpg) ++ List( new HTMLParserPass(cpg, sourceRepoLocation, ruleCache, privadoInputConfig = privadoInput), { if (privadoInput.assetDiscovery) { new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") @@ -77,6 +77,7 @@ class PythonProcessor( } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala b/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala index b4b452b35..bb2b7b82b 100644 --- a/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala @@ -23,26 +23,20 @@ package ai.privado.languageEngine.ruby.processor -import ai.privado.audit.{AuditReportEntryPoint, DEDSourceDiscovery} import ai.privado.cache.* -import ai.privado.entrypoint.ScanProcessor.config -import ai.privado.entrypoint.{PrivadoInput, ScanProcessor} -import ai.privado.exporter.{ExcelExporter, JSONExporter} -import ai.privado.exporter.monolith.MonolithExporter +import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.ruby.passes.* import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.languageEngine.ruby.passes.download.DownloadDependenciesPass -import ai.privado.languageEngine.ruby.passes.* import ai.privado.languageEngine.ruby.semantic.Language.* -import ai.privado.metric.MetricHandler -import ai.privado.model.Constants.{cpgOutputFileName, outputAuditFileName, outputDirectoryName, outputFileName} -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} -import ai.privado.passes.{DBTParserPass, ExperimentalLambdaDataFlowSupportPass, JsonPropertyParserPass, SQLParser} -import ai.privado.semantic.language.* +import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} +import ai.privado.passes.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} import better.files.File -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.rubysrc2cpg.deprecated.astcreation.ResourceManagedParser import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* @@ -58,28 +52,21 @@ import io.joern.x2cpg.passes.controlflow.cfgcreation.{Cfg, CfgCreator} import io.joern.x2cpg.passes.controlflow.cfgdominator.CfgDominatorPass import io.joern.x2cpg.passes.controlflow.codepencegraph.CdgPass import io.joern.x2cpg.passes.frontend.* -import io.joern.x2cpg.utils.ConcurrentTaskUtil import io.joern.x2cpg.{SourceFiles, ValidationMode, X2Cpg, X2CpgConfig} import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Cpg, Languages, Operators} +import io.shiftleft.codepropertygraph.generated.{Cpg, Languages} +import io.shiftleft.passes.CpgPassBase import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} -import org.slf4j.LoggerFactory import overflowdb.BatchedUpdate.DiffGraphBuilder -import ai.privado.dataflow.Dataflow -import ai.privado.cache.* -import ai.privado.languageEngine.base.processor.BaseProcessor -import io.shiftleft.passes.CpgPassBase import java.util -import java.util.Calendar -import java.util.concurrent.{Callable, Executors} import scala.collection.mutable.ListBuffer import scala.concurrent.* +import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.DurationLong import scala.util.{Failure, Success, Try, Using} -import scala.concurrent.ExecutionContext.Implicits.global class RubyProcessor( ruleCache: RuleCache, @@ -93,7 +80,8 @@ class RubyProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -107,7 +95,8 @@ class RubyProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { @@ -124,7 +113,7 @@ class RubyProcessor( // Using our own pass by overriding languageEngine's pass // new RubyImportResolverPass(cpg, packageTableInfo).createAndApply() val globalSymbolTable = new SymbolTable[LocalKey](SBKey.fromNodeToLocalKey) - passesList ++ List(new GlobalImportPass(cpg, globalSymbolTable)) ++ + super.applyPrivadoPasses(cpg) ++ passesList ++ List(new GlobalImportPass(cpg, globalSymbolTable)) ++ new PrivadoRubyTypeRecoveryPassGenerator(cpg, globalSymbolTable).generate() ++ List( new RubyTypeHintCallLinker(cpg), @@ -133,10 +122,10 @@ class RubyProcessor( new SQLParser(cpg, sourceRepoLocation, ruleCache), new DBTParserPass(cpg, sourceRepoLocation, ruleCache, databaseDetailsCache) ) - passesList } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/passes/DependencyNodePass.scala b/src/main/scala/ai/privado/passes/DependencyNodePass.scala new file mode 100644 index 000000000..c634786ed --- /dev/null +++ b/src/main/scala/ai/privado/passes/DependencyNodePass.scala @@ -0,0 +1,43 @@ +package ai.privado.passes + +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.tagger.PrivadoSimpleCpgPass +import ai.privado.utility.Utilities +import io.shiftleft.codepropertygraph.generated.nodes.{NewDependency, NewFile} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} + +import scala.collection.mutable + +/** This pass is using single threaded CPG pass mechanism. + * a. The contents to be processed are not going to be very huge as the list contains only the identified 3p + * dependencies only. b. We need to create a file node. Which can be repeated across the multiple nodes. If we use + * parallel mechanism. It will be difficult to handle that use case. + * + * @param cpg + * @param dependencies + * @param projectRoot + */ +class DependencyNodePass(cpg: Cpg, dependencies: List[DependencyInfo], projectRoot: String) + extends PrivadoSimpleCpgPass(cpg) + with Utility { + val fileNodeMap: mutable.Map[String, NewFile] = mutable.Map[String, NewFile]() + def run(builder: DiffGraphBuilder): Unit = { + dependencies.foreach(dependency => { + val dep = NewDependency() + .name(dependency.getFullDependencyName()) + .version(dependency.version) + .lineNumber(dependency.lineNumber) + .code(dependency.code) + + val fileNode = fileNodeMap.get(dependency.filePath) match { + case Some(fileNode) => fileNode + case None => + val fileNode = Utilities.addFileNode(dependency.filePath, builder) + fileNodeMap.put(dependency.filePath, fileNode) + fileNode + } + builder.addNode(dep) + builder.addEdge(dep, fileNode, EdgeTypes.SOURCE_FILE) + }) + } +} diff --git a/src/main/scala/ai/privado/passes/JsonParser.scala b/src/main/scala/ai/privado/passes/JsonParser.scala new file mode 100644 index 000000000..f90f2157f --- /dev/null +++ b/src/main/scala/ai/privado/passes/JsonParser.scala @@ -0,0 +1,48 @@ +package ai.privado.passes + +import io.circe.Json +import io.circe.parser.parse +import org.slf4j.LoggerFactory +import better.files.File + +trait JsonParser { + + /** Parses a JSON file and returns a list of key-value pairs for properties related to database connections and API + * endpoints. + * + * @param file + * the path to the JSON file to parse + * @return + * a list of key-value pairs where the keys match either the database connection or API endpoint naming conventions + */ + def getJSONKeyValuePairs(file: String): List[(String, String)] = { + val json = parse(File(file).contentAsString) + + // Recursively scan through the JSON to extract out all keys + def extractKeyValuePairs(json: Json, prefix: String = ""): List[(String, String)] = { + json match { + case obj if obj.isObject => + obj.asObject.get.toMap.toList.flatMap { case (key, value) => + val newPrefix = if (prefix.isEmpty) key else s"$prefix.$key" + extractKeyValuePairs(value, newPrefix) + } + case arr if arr.isArray => + arr.asArray.get.toList.zipWithIndex.flatMap { case (value, index) => + val newPrefix = s"$prefix[$index]" + extractKeyValuePairs(value, newPrefix) + } + case other => + List((prefix, other.asString.getOrElse(other.toString))) + } + } + + val keyValuePairs = json match { + case Right(jsonObject) => extractKeyValuePairs(jsonObject) + case Left(parsingError) => + List.empty + } + + keyValuePairs + } + +} diff --git a/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala b/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala index 3c6754881..f1faeccc7 100644 --- a/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala +++ b/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala @@ -15,9 +15,10 @@ import better.files.File import scala.collection.mutable import scala.io.Source import scala.util.{Try, Using} -class JsonPropertyParserPass(cpg: Cpg, projectRoot: String) extends PrivadoParallelCpgPass[String](cpg) { +class JsonPropertyParserPass(cpg: Cpg, projectRoot: String) + extends PrivadoParallelCpgPass[String](cpg) + with JsonParser { - val logger = LoggerFactory.getLogger(getClass) override def generateParts(): Array[String] = { val files = Try(File(projectRoot).listRecursively.filter(_.isRegularFile).map(_.path.toString).toArray).toOption @@ -40,44 +41,4 @@ class JsonPropertyParserPass(cpg: Cpg, projectRoot: String) extends PrivadoParal builder.addNode(propertyNode) propertyNode } - - /** Parses a JSON file and returns a list of key-value pairs for properties related to database connections and API - * endpoints. - * - * @param file - * the path to the JSON file to parse - * @return - * a list of key-value pairs where the keys match either the database connection or API endpoint naming conventions - */ - private def getJSONKeyValuePairs(file: String): List[(String, String)] = { - import better.files.File - val json = parse(File(file).contentAsString) - - // Recursively scan through the JSON to extract out all keys - def extractKeyValuePairs(json: Json, prefix: String = ""): List[(String, String)] = { - json match { - case obj if obj.isObject => - obj.asObject.get.toMap.toList.flatMap { case (key, value) => - val newPrefix = if (prefix.isEmpty) key else s"$prefix.$key" - extractKeyValuePairs(value, newPrefix) - } - case arr if arr.isArray => - arr.asArray.get.toList.zipWithIndex.flatMap { case (value, index) => - val newPrefix = s"$prefix[$index]" - extractKeyValuePairs(value, newPrefix) - } - case other => - List((prefix, other.asString.getOrElse(other.toString))) - } - } - - val keyValuePairs = json match { - case Right(jsonObject) => extractKeyValuePairs(jsonObject) - case Left(parsingError) => - logger.debug(parsingError.toString) - List.empty - } - - keyValuePairs - } } diff --git a/src/main/scala/ai/privado/passes/PropertyParserPass.scala b/src/main/scala/ai/privado/passes/PropertyParserPass.scala index eaa491744..eed01dbd1 100644 --- a/src/main/scala/ai/privado/passes/PropertyParserPass.scala +++ b/src/main/scala/ai/privado/passes/PropertyParserPass.scala @@ -1,4 +1,4 @@ -package ai.privado.utility +package ai.privado.passes import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.NewJavaProperty @@ -30,13 +30,14 @@ import org.yaml.snakeyaml.nodes.{MappingNode, Node, NodeTuple, ScalarNode, Seque import scala.jdk.CollectionConverters.* import ai.privado.model.{Constants, Language} import ai.privado.tagger.PrivadoParallelCpgPass +import ai.privado.utility.{ConfigParserUtility, Utilities} import org.yaml.snakeyaml.constructor.SafeConstructor import better.files.File.VisitOptions import java.nio.file.Path import scala.collection.mutable.ListBuffer -object FileExtensions { +object PropertyFileExtensions { val PROPERTIES = ".properties" val YAML = ".yaml" val YML = ".yml" @@ -54,7 +55,8 @@ class PropertyParserPass( language: Language.Value, propertyFilterCache: PropertyFilterCache = PropertyFilterCache(), privadoInput: PrivadoInput = PrivadoInput() -) extends PrivadoParallelCpgPass[String](cpg) { +) extends PrivadoParallelCpgPass[String](cpg) + with Utility { val PLACEHOLDER_TOKEN_START_END = "@@" val logger = LoggerFactory.getLogger(getClass) @@ -64,11 +66,11 @@ class PropertyParserPass( configFiles( projectRoot, Set( - FileExtensions.PROPERTIES, - FileExtensions.YAML, - FileExtensions.YML, - FileExtensions.XML, - FileExtensions.CONF + PropertyFileExtensions.PROPERTIES, + PropertyFileExtensions.YAML, + PropertyFileExtensions.YML, + PropertyFileExtensions.XML, + PropertyFileExtensions.CONF ) ).toArray } @@ -76,30 +78,40 @@ class PropertyParserPass( if (privadoInput.enableIngressAndEgressUrls) { configFiles( projectRoot, - Set(FileExtensions.JSON, FileExtensions.ENV, FileExtensions.YAML, FileExtensions.YML) + Set( + PropertyFileExtensions.JSON, + PropertyFileExtensions.ENV, + PropertyFileExtensions.YAML, + PropertyFileExtensions.YML + ) ).toArray } else { - configFiles(projectRoot, Set(FileExtensions.JSON, FileExtensions.ENV)).toArray + configFiles(projectRoot, Set(PropertyFileExtensions.JSON, PropertyFileExtensions.ENV)).toArray } case Language.PYTHON => configFiles( projectRoot, - Set(FileExtensions.INI, FileExtensions.ENV, FileExtensions.YAML, FileExtensions.YML) + Set( + PropertyFileExtensions.INI, + PropertyFileExtensions.ENV, + PropertyFileExtensions.YAML, + PropertyFileExtensions.YML + ) ).toArray case Language.RUBY => - (configFiles(projectRoot, Set(FileExtensions.ENV)) ++ configFiles( + (configFiles(projectRoot, Set(PropertyFileExtensions.ENV)) ++ configFiles( projectRoot, - Set(FileExtensions.YML, FileExtensions.YAML) + Set(PropertyFileExtensions.YML, PropertyFileExtensions.YAML) ).filter(_.matches(".*(settings|config).*"))).toArray // Ruby has a lot of yaml files so creating property nodes for all of them, exposes a lot of property nodes, // which are incorrect, so we go by the approach of being selective and creating property nodes for only the impacted files case Language.GO => - (configFiles(projectRoot, Set(FileExtensions.YAML, FileExtensions.YML))).toArray + (configFiles(projectRoot, Set(PropertyFileExtensions.YAML, PropertyFileExtensions.YML))).toArray } } override def runOnPart(builder: DiffGraphBuilder, file: String): Unit = { - val fileNode = addFileNode(file, builder) + val fileNode = addFileNode(file, builder, projectRoot) val propertyNodes = obtainKeyValuePairs(file, builder).map(pair => addPropertyNode(pair, builder)) propertyNodes.foreach(builder.addEdge(_, fileNode, EdgeTypes.SOURCE_FILE)) } @@ -354,13 +366,6 @@ class PropertyParserPass( propertyNode } - private def addFileNode(name: String, builder: BatchedUpdate.DiffGraphBuilder): NewFile = { - val relativeFileName = Path.of(projectRoot).relativize(Path.of(name)).toString - val fileNode = NewFile().name(relativeFileName) - builder.addNode(fileNode) - fileNode - } - private def configFiles(projectRoot: String, extensions: Set[String]): List[String] = { def getListOfFiles(dir: String): List[File] = { val d = new File(dir) diff --git a/src/main/scala/ai/privado/passes/Utility.scala b/src/main/scala/ai/privado/passes/Utility.scala new file mode 100644 index 000000000..99151007b --- /dev/null +++ b/src/main/scala/ai/privado/passes/Utility.scala @@ -0,0 +1,21 @@ +package ai.privado.passes + +import io.shiftleft.codepropertygraph.generated.nodes.NewFile +import overflowdb.BatchedUpdate + +import java.nio.file.Path + +trait Utility { + + def addFileNode(name: String, builder: BatchedUpdate.DiffGraphBuilder, projectRoot: String): NewFile = { + val relativeFileName = Path.of(projectRoot).relativize(Path.of(name)).toString + val fileNode = NewFile().name(relativeFileName) + builder.addNode(fileNode) + fileNode + } + + def combinedRulePattern(patterns: List[String]): String = { + patterns.mkString("(", "|", ")") + } + +} diff --git a/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala b/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala new file mode 100644 index 000000000..9c99072dd --- /dev/null +++ b/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala @@ -0,0 +1,34 @@ +package ai.privado.tagger.sink + +import ai.privado.cache.RuleCache +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.passes.Utility +import ai.privado.tagger.PrivadoSimpleCpgPass +import ai.privado.utility.Utilities +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* +import org.slf4j.LoggerFactory + +class DependencyNodeTagger(cpg: Cpg, dependencies: List[DependencyInfo], ruleCache: RuleCache) + extends PrivadoSimpleCpgPass(cpg) + with Utility { + private val logger = LoggerFactory.getLogger(this.getClass) + def run(builder: DiffGraphBuilder): Unit = { + dependencies.foreach(dependency => { + cpg.dependency + .nameExact(dependency.getFullDependencyName()) + .where(_.file.nameExact(dependency.filePath)) + .headOption match { + case Some(dep) => + ruleCache.getRuleInfo(dependency.ruleId) match { + case Some(rule) => Utilities.addRuleTags(builder, dep, rule, ruleCache) + case None => + logger.error( + s"Dynamic rule ${dependency.ruleId} is not found inside rule cache. It seems given rule is not passed" + ) + } + case None => + } + }) + } +} diff --git a/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala b/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala index 89f94777e..07faaa135 100644 --- a/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala +++ b/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala @@ -2,7 +2,7 @@ package ai.privado.exporter import ai.privado.cache.{PropertyFilterCache, RuleCache} import ai.privado.model.{ConfigAndRules, Language, SystemConfig} -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala b/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala index 9cbfdc29d..4ffede554 100644 --- a/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala +++ b/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala @@ -2,7 +2,7 @@ package ai.privado.exporter import ai.privado.cache.RuleCache import ai.privado.model.{ConfigAndRules, Language, SystemConfig} -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/inputprocessor/DependencyTaggingProcessorTest.scala b/src/test/scala/ai/privado/inputprocessor/DependencyTaggingProcessorTest.scala new file mode 100644 index 000000000..504a8bbaa --- /dev/null +++ b/src/test/scala/ai/privado/inputprocessor/DependencyTaggingProcessorTest.scala @@ -0,0 +1,60 @@ +package ai.privado.inputprocessor + +import better.files.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class DependencyTaggingProcessorTest + extends AnyWordSpec + with Matchers + with BeforeAndAfterAll + with DependencyTaggingProcessor { + + "Parsing test" should { + "Working test" in { + File.usingTemporaryDirectory("deptest") { tmpDir => + val tempFile = tmpDir / "java.json"; + tempFile.writeText(""" + |[ + | { + | "groupId": "com.privado.sample", + | "dependencyName": "Scanner", + | "version": "1.0", + | "code" : "Scanner", + | "ruleId": "ThirdParties.SDK.Scanner", + | "ruleName": "Privado Scanner", + | "ruleDomains": [ + | "scanner.privado.ai" + | ], + | "ruleTags": [], + | "lineNumber": 10, + | "filePath": "/path/pom.xml" + | } + |] + |""".stripMargin) + val dependencies: List[DependencyInfo] = parseDependencyInfo(tempFile.pathAsString) + dependencies shouldBe List( + DependencyInfo( + "com.privado.sample", + "Scanner", + "1.0", + "Scanner", + "ThirdParties.SDK.Scanner", + "Privado Scanner", + List("scanner.privado.ai"), + List(), + 10, + "/path/pom.xml" + ) + ) + } + } + + "Graceful handling of error situation" in { + val dependencies: List[DependencyInfo] = parseDependencyInfo("path/nonexistingfile.json") + dependencies shouldBe List() + } + } + +} diff --git a/src/test/scala/ai/privado/entrypoint/DynamicRuleMergerTest.scala b/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala similarity index 99% rename from src/test/scala/ai/privado/entrypoint/DynamicRuleMergerTest.scala rename to src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala index b73cdbab3..7cf21b37e 100644 --- a/src/test/scala/ai/privado/entrypoint/DynamicRuleMergerTest.scala +++ b/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala @@ -1,4 +1,4 @@ -package ai.privado.entrypoint +package ai.privado.inputprocessor import ai.privado.cache.RuleCache import ai.privado.model.{CatLevelOne, ConfigAndRules, Constants, FilterProperty, Language, NodeType, RuleInfo} diff --git a/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala b/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala index cf90fdd70..e68697e46 100644 --- a/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala @@ -4,28 +4,19 @@ import ai.privado.cache.{AppCache, FileLinkingMetadata, RuleCache, TaggerCache} import ai.privado.entrypoint.PrivadoInput import ai.privado.languageEngine.go.tagger.sink.GoAPITagger import ai.privado.languageEngine.go.tagger.source.IdentifierTagger -import ai.privado.model.{ - CatLevelOne, - ConfigAndRules, - Constants, - FilterProperty, - Language, - NodeType, - RuleInfo, - SystemConfig -} -import ai.privado.utility.PropertyParserPass +import ai.privado.model.* +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.gosrc2cpg.{Config, GoSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import io.shiftleft.semanticcpg.language.* -import ai.privado.semantic.language.* abstract class GoYamlLinkerPassTest extends GoYamlFileLinkerPassTestBase { override val yamlFileContents: String = """ diff --git a/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala b/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala index af2482526..29e73e83a 100644 --- a/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala @@ -2,11 +2,12 @@ package ai.privado.languageEngine.java.passes.config import ai.privado.cache.{AppCache, FileLinkingMetadata, RuleCache, TaggerCache} import ai.privado.entrypoint.PrivadoInput -import ai.privado.semantic.language.* import ai.privado.languageEngine.java.tagger.sink.api.JavaAPITagger import ai.privado.languageEngine.java.tagger.source.* import ai.privado.model.* -import ai.privado.utility.{PropertyParserPass, StatsRecorder} +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} diff --git a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala index 430cfc9a9..982f3f182 100644 --- a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala @@ -25,21 +25,20 @@ package ai.privado.languageEngine.java.passes.config import ai.privado.cache.{AppCache, RuleCache} import ai.privado.entrypoint.PrivadoInput +import ai.privado.exporter.HttpConnectionMetadataExporter +import ai.privado.model.{Constants, Language} +import ai.privado.passes.PropertyParserPass import ai.privado.semantic.language.* -import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.testfixtures.JavaFrontendTestSuite import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, JavaProperty, Literal, Method, MethodParameterIn} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import ai.privado.exporter.HttpConnectionMetadataExporter -import ai.privado.testfixtures.JavaFrontendTestSuite -import ai.privado.model.Constants class AnnotationTests extends JavaFrontendTestSuite { "ConfigFilePass" should { diff --git a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala index f0e438e41..91e9be94b 100644 --- a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala @@ -2,16 +2,16 @@ package ai.privado.languageEngine.java.passes.config import ai.privado.cache.{AppCache, RuleCache} import ai.privado.model.{ConfigAndRules, Language, SystemConfig} -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import io.shiftleft.semanticcpg.language.* -import ai.privado.semantic.language.* import scala.collection.mutable diff --git a/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala b/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala new file mode 100644 index 000000000..445f8995b --- /dev/null +++ b/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala @@ -0,0 +1,459 @@ +package ai.privado.languageEngine.java.tagger.sink + +import ai.privado.cache.RuleCache +import ai.privado.exporter.SinkExporterValidator +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.model +import ai.privado.model.* +import ai.privado.model.exporter.{DataFlowSubCategoryPathExcerptModel, SinkProcessingModel} +import ai.privado.testfixtures.JavaFrontendTestSuite + +import java.io.File +import scala.collection.mutable + +class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValidator { + + "Java pom.xml simple use case" should { + val dynamicRule = List( + RuleInfo( + "ThirdParties.SDK.twilio", + "twilio", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("www.twilio.com"), + List(".*(com.twilio.sdk).*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ), + RuleInfo( + "ThirdParties.SDK.Google.Sheets", + "Google Sheets", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("sheets.google.com"), + List(".*(com.google.apis).*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ) + ) + + val configAndRule = ConfigAndRules(sinks = dynamicRule) + val ruleCache = RuleCache().setRule(configAndRule) + val twilioDep = List( + DependencyInfo( + "com.twilio.sdk", + "twilio", + "8.19.1", + "twilio", + "ThirdParties.SDK.twilio", + "Twilio", + List("www.twilio.com"), + List(), + 32, + "pom.xml" + ), + DependencyInfo( + "com.google.apis", + "google-api-services-sheets", + "v4-rev493-1.23.0", + "google-api-services-sheets", + "ThirdParties.SDK.Google.Sheets", + "Google Sheets", + List("sheets.google.com"), + List(), + 42, + "pom.xml" + ) + ) + val cpg = code( + """ + | + | + | 4.0.0 + | + | org.springframework.boot + | spring-boot-starter-parent + | 2.5.4 + | + | + | com.atlan + | SMSService + | 0.0.1-SNAPSHOT + | SMSService + | Demo project for Spring Boot + | + | 1.8 + | 2020.0.3 + | + | + | + | mysql + | mysql-connector-java + | + | + | org.springframework.boot + | spring-boot-starter-web + | + | + | com.twilio.sdk + | twilio + | 8.19.1 + | + | + | org.springframework.boot + | spring-boot-starter-test + | test + | + | + | com.google.apis + | google-api-services-sheets + | v4-rev493-1.23.0 + | + | + | + | + | + | org.springframework.boot + | spring-boot-maven-plugin + | + | + | + | + |""".stripMargin, + "pom.xml" + ).withDependencies(twilioDep).withRuleCache(ruleCache) + + "3p dependency should get added in processing section" in { + val outjson = cpg.getPrivadoJson() + getSinks(outjson).size shouldBe 2 + val sinkProcessing = getSinkProcessings(outjson) + val sinkProceMap: mutable.Map[String, SinkProcessingModel] = mutable.Map[String, SinkProcessingModel]() + sinkProcessing.size shouldBe 2 + sinkProcessing.foreach(sink => { + sinkProceMap.put(sink.sinkId, sink) + }) + + val twilioSink = sinkProceMap.get("ThirdParties.SDK.twilio") + twilioSink shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.twilio", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = "twilio", + lineNumber = 32, + columnNumber = -1, + fileName = "pom.xml", + excerpt = + "\t\t\torg.springframework.boot\n\t\t\tspring-boot-starter-web\n\t\t\n\t\t\n\t\t\tcom.twilio.sdk\n\t\t\ttwilio /* <=== */ \n\t\t\t8.19.1\n\t\t\n\t\t\n\t\t\torg.springframework.boot\n\t\t\tspring-boot-starter-test", + arguments = None + ) + ) + ) + ) + + val gsheetSink = sinkProceMap.get("ThirdParties.SDK.Google.Sheets") + gsheetSink shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Google.Sheets", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = "google-api-services-sheets", + lineNumber = 42, + columnNumber = -1, + fileName = "pom.xml", + excerpt = + "\t\t\tspring-boot-starter-test\n\t\t\ttest\n\t\t\n \t\t\n\t\t\tcom.google.apis\n\t\t\tgoogle-api-services-sheets /* <=== */ \n\t\t\tv4-rev493-1.23.0\n\t\t\n\t\n\t\n\t\t", + arguments = None + ) + ) + ) + ) + } + } + + "Gradle tests" should { + val dynamicRule = List( + RuleInfo( + "ThirdParties.SDK.Google.Play.Services", + "Google Play Services", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("developers.google.com"), + List("(?i).*com.google.android.gms.*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ), + RuleInfo( + "ThirdParties.SDK.Firebase.Authentication", + "Firebase Authentication", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("firebase.google.com"), + List("(?i).*com.google.firebase.*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ), + RuleInfo( + "ThirdParties.SDK.Firebase", + "Firebase", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("firebase.google.com"), + List("(?i).*com.google.firebase.*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ) + ) + + val configAndRule = ConfigAndRules(sinks = dynamicRule) + val ruleCache = RuleCache().setRule(configAndRule) + val twilioDep = List( + DependencyInfo( + "com.google.android.gms", + "play-services-auth", + "17.0.0", + " implementation 'com.google.android.gms:play-services-auth:17.0.0'\n", + "ThirdParties.SDK.Google.Play.Services", + "Google Play Services", + List("developers.google.com"), + List(), + 36, + "app/build.gradle" + ), + DependencyInfo( + "com.google.firebase", + "firebase-auth", + "19.0.0", + " implementation 'com.google.firebase:firebase-auth:19.0.0'\n", + "ThirdParties.SDK.Firebase.Authentication", + "Firebase Authentication", + List("firebase.google.com"), + List(), + 35, + "app/build.gradle" + ), + DependencyInfo( + "com.google.firebase", + "firebase-core", + "17.1.0", + " implementation 'com.google.firebase:firebase-core:17.1.0'\n", + "ThirdParties.SDK.Firebase", + "Firebase", + List("firebase.google.com"), + List(), + 32, + "app/build.gradle" + ), + DependencyInfo( + "com.google.gms", + "google-services", + "4.2.0", + " classpath 'com.google.gms:google-services:4.2.0'\n", + "ThirdParties.SDK.Google.Play.Services", + "Google Play Services", + List("play.google.com"), + List(), + 11, + "build.gradle" + ) + ) + val cpg = code( + """ + |buildscript { + | repositories { + | google() + | jcenter() + | + | } + | dependencies { + | classpath 'com.android.tools.build:gradle:3.5.3' + | classpath 'com.google.gms:google-services:4.2.0' + | + | // NOTE: Do not place your application dependencies here; they belong + | // in the individual module build.gradle files + | } + |} + | + |allprojects { + | repositories { + | google() + | jcenter() + | + | } + |} + | + |task clean(type: Delete) { + | delete rootProject.buildDir + |} + |""".stripMargin, + "build.gradle" + ).moreCode( + """ + |apply plugin: 'com.android.application' + |apply plugin: 'com.google.gms.google-services' + | + |android { + | compileSdkVersion 29 + | buildToolsVersion "29.0.1" + | defaultConfig { + | applicationId "com.example.edu" + | minSdkVersion 22 + | targetSdkVersion 29 + | versionCode 1 + | versionName "1.0" + | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + | } + | buildTypes { + | release { + | minifyEnabled false + | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + | } + | } + |} + | + |dependencies { + | implementation fileTree(dir: 'libs', include: ['*.jar']) + | implementation 'androidx.appcompat:appcompat:1.0.2' + | implementation 'androidx.constraintlayout:constraintlayout:1.1.3' + | implementation 'androidx.cardview:cardview:1.0.0' + | implementation 'com.android.support:appcompat-v7:29.0.0' + | implementation 'com.android.support:design:29.0.1' + | implementation 'androidx.appcompat:appcompat:1.0.0' + | implementation 'androidx.core:core:1.0.0' + | implementation 'com.google.firebase:firebase-core:17.1.0' + | implementation 'com.google.firebase:firebase-auth:19.0.0' + | implementation 'com.google.firebase:firebase-database:19.0.0' + | implementation 'com.google.firebase:firebase-auth:19.0.0' + | implementation 'com.google.android.gms:play-services-auth:17.0.0' + | implementation 'com.github.d-max:spots-dialog:1.1@aar' + | implementation 'com.google.android.material:material:1.0.0' + | implementation 'com.android.volley:volley:1.1.1' + | implementation 'com.google.firebase:firebase-messaging:20.1.0' + | implementation 'br.com.simplepass:loading-button-android:1.14.0' + | implementation 'com.google.android.material:material:1.0.0' + | implementation 'com.google.android.material:material:1.1.0-alpha09' + | implementation 'com.google.firebase:firebase-storage:16.0.4' + | implementation 'androidx.navigation:navigation-fragment:2.0.0' + | implementation 'androidx.navigation:navigation-ui:2.0.0' + | implementation 'androidx.lifecycle:lifecycle-extensions:2.0.0' + | + | testImplementation 'junit:junit:4.12' + | androidTestImplementation 'androidx.test:runner:1.2.0' + | androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + |} + |""".stripMargin, + List("app", "build.gradle").mkString(File.separator) + ).withDependencies(twilioDep) + .withRuleCache(ruleCache) + + "3p dependency should get added in processing section" in { + val outjson = cpg.getPrivadoJson() + getSinks(outjson).size shouldBe 3 + val sinkProcessing = getSinkProcessings(outjson) + val sinkProceMap: mutable.Map[String, SinkProcessingModel] = mutable.Map[String, SinkProcessingModel]() + sinkProcessing.size shouldBe 3 + sinkProcessing.foreach(sink => { + sinkProceMap.put(sink.sinkId, sink) + }) + + val playServices = sinkProceMap.get("ThirdParties.SDK.Google.Play.Services") + playServices shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Google.Play.Services", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = " classpath 'com.google.gms:google-services:4.2.0'\n", + lineNumber = 11, + columnNumber = -1, + fileName = "build.gradle", + excerpt = + " \n }\n dependencies {\n classpath 'com.android.tools.build:gradle:3.5.3'\n classpath 'com.google.gms:google-services:4.2.0'\n /* <=== */ \n // NOTE: Do not place your application dependencies here; they belong\n // in the individual module build.gradle files\n }\n}\n", + arguments = None + ), + DataFlowSubCategoryPathExcerptModel( + sample = " implementation 'com.google.android.gms:play-services-auth:17.0.0'\n", + lineNumber = 36, + columnNumber = -1, + fileName = "app/build.gradle", + excerpt = + " implementation 'androidx.appcompat:appcompat:1.0.0'\n implementation 'androidx.core:core:1.0.0'\n implementation 'com.google.firebase:firebase-core:17.1.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.firebase:firebase-database:19.0.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0' /* <=== */ \n implementation 'com.google.android.gms:play-services-auth:17.0.0'\n implementation 'com.github.d-max:spots-dialog:1.1@aar'\n implementation 'com.google.android.material:material:1.0.0'\n implementation 'com.android.volley:volley:1.1.1'\n implementation 'com.google.firebase:firebase-messaging:20.1.0'", + arguments = None + ) + ) + ) + ) + + val firebaseAuth = sinkProceMap.get("ThirdParties.SDK.Firebase.Authentication") + firebaseAuth shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Firebase.Authentication", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = " implementation 'com.google.firebase:firebase-auth:19.0.0'\n", + lineNumber = 35, + columnNumber = -1, + fileName = "app/build.gradle", + excerpt = + " implementation 'com.android.support:design:29.0.1'\n implementation 'androidx.appcompat:appcompat:1.0.0'\n implementation 'androidx.core:core:1.0.0'\n implementation 'com.google.firebase:firebase-core:17.1.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.firebase:firebase-database:19.0.0' /* <=== */ \n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.android.gms:play-services-auth:17.0.0'\n implementation 'com.github.d-max:spots-dialog:1.1@aar'\n implementation 'com.google.android.material:material:1.0.0'\n implementation 'com.android.volley:volley:1.1.1'", + arguments = None + ) + ) + ) + ) + + val firebase = sinkProceMap.get("ThirdParties.SDK.Firebase") + firebase shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Firebase", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = " implementation 'com.google.firebase:firebase-core:17.1.0'\n", + lineNumber = 32, + columnNumber = -1, + fileName = "app/build.gradle", + excerpt = + " implementation 'androidx.constraintlayout:constraintlayout:1.1.3'\n implementation 'androidx.cardview:cardview:1.0.0'\n implementation 'com.android.support:appcompat-v7:29.0.0'\n implementation 'com.android.support:design:29.0.1'\n implementation 'androidx.appcompat:appcompat:1.0.0'\n implementation 'androidx.core:core:1.0.0' /* <=== */ \n implementation 'com.google.firebase:firebase-core:17.1.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.firebase:firebase-database:19.0.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.android.gms:play-services-auth:17.0.0'", + arguments = None + ) + ) + ) + ) + } + } +} diff --git a/src/test/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJSTests.scala b/src/test/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJSTests.scala new file mode 100644 index 000000000..62585c4ea --- /dev/null +++ b/src/test/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJSTests.scala @@ -0,0 +1,425 @@ +package ai.privado.languageEngine.javascript.metadata + +import ai.privado.cache.FileLinkingMetadata +import ai.privado.testfixtures.JavaScriptBaseCpgFrontendTestSuite + +import scala.collection.immutable.Set + +class FileImportMappingPassJSTests extends JavaScriptBaseCpgFrontendTestSuite { + + "File import mapping in javascript" should { + "resolve import files case 1" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from './module.js'; + |functionName(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from module'); + |} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 2" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import defaultExport from './module.js'; + |defaultExport(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export default function defaultExport() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 3" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import * as module from './module.js'; + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() {} + |export const someValue = 42; + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 4" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |const module = require('./module.js'); + |module(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports = function() { + | console.log('Function from module'); + |}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 5" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |const { functionName } = require('./module.js'); + |functionName(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports.functionName = function() {}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 6" ignore { + // TODO Failing as no import node created + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |require(['./module'], function(module) { + | module.functionName(); + |}); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports.functionName = function() {}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 7" ignore { + // TODO Failing as no import node created + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import('./module.js').then(module => { + | module.functionName(); + |}); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports.functionName = function() {}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 8" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from 'common/module'; + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 9" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from 'common/module'; + | + |""".stripMargin, + "src/util.js" + ).moreCode( + """ + |export function functionName() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 10" ignore { + // TODO Failing as no import node created + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |const functionName = () => import(/* webpackChunkName: "module" */ 'common/module'); + | + |""".stripMargin, + "src/util.js" + ).moreCode( + """ + |export function functionName() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 11" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from '../module.js'; + |functionName(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from module'); + |} + | + |""".stripMargin, + "src/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/module.js") + } + } + + "File import mapping in TypeScript" should { + "resolve import files case 1" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from './module'; + |functionName(); + | + |""".stripMargin, + "src/common/util.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from module'); + |} + | + |""".stripMargin, + "src/common/module.ts" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 2" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import type { TypeName } from './module'; + | + |""".stripMargin, + "src/common/util.ts" + ).moreCode( + """ + |export type TypeName = { /* ... */ }; + | + |""".stripMargin, + "src/common/module.ts" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 3" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "common/module.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/module": ["common/module"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("common/module.ts") + } + + "resolve import files case 4" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "src/common/module.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/*": ["src/common/*"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 5" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import * as module from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "src/common/module.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/*": ["src/common/*"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 6" ignore { + // TODO Need to work on getting the export information from default CPG + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName1 } from '@utils/module'; + |import { functionName2 } from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName1() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "common/module/module1.ts" + ).moreCode( + """ + |export function functionName2() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "common/module/module2.ts" + ).moreCode( + """ + |export * from './module1'; + |export * from './module2' + |""".stripMargin, + "common/module/index.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/module": ["common/module"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("common/module/module1.ts") + } + } + +} diff --git a/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala index c69800b9d..c3d698cec 100644 --- a/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala @@ -1,16 +1,16 @@ package ai.privado.languageEngine.javascript.passes.config import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} -import io.joern.jssrc2cpg.{Config, JsSrc2Cpg} import io.joern.jssrc2cpg.passes.ImportsPass +import io.joern.jssrc2cpg.{Config, JsSrc2Cpg} import io.joern.x2cpg.X2Cpg import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers diff --git a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala index d5dbe3d45..b7b828745 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala @@ -1,12 +1,12 @@ package ai.privado.languageEngine.ruby.config import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* -import ai.privado.semantic.* import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.model.Language +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.* +import ai.privado.semantic.language.* import ai.privado.testfixtures.RubyFrontendTestSuite -import ai.privado.utility.PropertyParserPass import better.files.File import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala index 656b86768..41ee3fa07 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala @@ -3,7 +3,7 @@ package ai.privado.languageEngine.ruby.config import ai.privado.cache.RuleCache import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass import better.files.File import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala index 92d5468ed..314aeb389 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala @@ -1,11 +1,11 @@ package ai.privado.languageEngine.ruby.config import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* -import ai.privado.semantic.* import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.* +import ai.privado.semantic.language.* import better.files.File import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala index bf8772e49..662ffef89 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala @@ -1,14 +1,6 @@ package ai.privado.languageEngine.ruby.monolith -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - RuleCache, - S3DatabaseDetailsCache, - TaggerCache -} +import ai.privado.cache.* import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.PrivadoInput import ai.privado.exporter.monolith.MonolithExporter @@ -17,17 +9,17 @@ import ai.privado.languageEngine.go.tagger.source.IdentifierTagger import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.languageEngine.ruby.tagger.monolith.MonolithTagger import ai.privado.model.{Constants, Language} +import ai.privado.passes.PropertyParserPass import ai.privado.rule.RuleInfoTestData -import ai.privado.utility.PropertyParserPass import better.files.File import io.joern.dataflowengineoss.language.Path import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import io.shiftleft.semanticcpg.language.* import scala.collection.mutable diff --git a/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala index 9bc9e87d5..7cb65d806 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala @@ -2,20 +2,12 @@ package ai.privado.languageEngine.ruby.tagger.schema import ai.privado.cache.{DatabaseDetailsCache, PropertyFilterCache, RuleCache} import ai.privado.languageEngine.ruby.RubyTestBase.* +import ai.privado.model.* +import ai.privado.passes.PropertyParserPass +import ai.privado.rule.RuleInfoTestData import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import ai.privado.model.{ - ConfigAndRules, - DatabaseColumn, - DatabaseDetails, - DatabaseSchema, - DatabaseTable, - Language, - SourceCodeModel -} -import ai.privado.rule.RuleInfoTestData -import ai.privado.utility.PropertyParserPass class RubyMongoSchemaMapperTest extends AnyWordSpec with Matchers with BeforeAndAfterAll { diff --git a/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala b/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala index bdbddd4f5..b0c189cad 100644 --- a/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala +++ b/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala @@ -1,19 +1,19 @@ package ai.privado.passes import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* +import ai.privado.model.Language import ai.privado.model.sql.SQLQueryType +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File +import io.joern.console.cpgcreation.guessLanguage import io.joern.jssrc2cpg.Config import io.joern.x2cpg.X2Cpg import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import ai.privado.utility.PropertyParserPass -import ai.privado.model.Language -import io.joern.console.cpgcreation.guessLanguage -import io.shiftleft.semanticcpg.language._ class SQLPropertyParserTest extends AnyWordSpec with Matchers with BeforeAndAfterAll { "SQL Property parser" should { diff --git a/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala index 57c52c45f..021c38760 100644 --- a/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.c.processor.CProcessor import ai.privado.model.Language @@ -24,7 +17,9 @@ class TestCpgWithC(val fileSuffix: String, val language: Language.Value) extends s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new CProcessor( ruleCache, @@ -36,8 +31,9 @@ class TestCpgWithC(val fileSuffix: String, val language: Language.Value) extends appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata ) } } diff --git a/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala index d57d67500..c43df27c7 100644 --- a/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.csharp.processor.CSharpProcessor import ai.privado.model.Language @@ -24,7 +17,9 @@ class TestCpgWithCSharp(val fileSuffix: String, val language: Language.Value) ex s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new CSharpProcessor( ruleCache, @@ -36,8 +31,10 @@ class TestCpgWithCSharp(val fileSuffix: String, val language: Language.Value) ex appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala index a94093968..48fc965c9 100644 --- a/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.default.processor.DefaultProcessor import ai.privado.model.Language @@ -24,7 +17,9 @@ class TestCpgWithDefaultLanguage(val fileSuffix: String, val language: Language. s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new DefaultProcessor( ruleCache, @@ -36,8 +31,9 @@ class TestCpgWithDefaultLanguage(val fileSuffix: String, val language: Language. appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata ) } } diff --git a/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala index a17a33556..1b3027e7b 100644 --- a/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala @@ -1,18 +1,11 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.go.processor.GoProcessor -import ai.privado.model.{CatLevelOne, Constants, FilterProperty, Language, NodeType, RuleInfo} +import ai.privado.model.* import ai.privado.utility.StatsRecorder class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extends TestCpg { @@ -25,7 +18,9 @@ class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extend s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new GoProcessor( ruleCache, @@ -37,8 +32,10 @@ class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extend appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala index 7d698be5c..0beec502d 100644 --- a/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala @@ -2,10 +2,11 @@ package ai.privado.testfixtures import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.java.processor.JavaProcessor -import ai.privado.utility.StatsRecorder import ai.privado.model.Language +import ai.privado.utility.StatsRecorder class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) extends TestCpg { protected def getLanguageProcessor( @@ -16,7 +17,9 @@ class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) exte s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new JavaProcessor( ruleCache, @@ -28,8 +31,10 @@ class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) exte appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala index 9386aba21..a4fe80d53 100644 --- a/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala @@ -2,9 +2,10 @@ package ai.privado.testfixtures import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor -import ai.privado.languageEngine.javascript.processor.JavascriptProcessor -import ai.privado.model.{CatLevelOne, Constants, FilterProperty, Language, NodeType, RuleInfo} +import ai.privado.languageEngine.javascript.processor.{JavascriptProcessor, JavascriptBaseCPGProcessor} +import ai.privado.model.* import ai.privado.utility.StatsRecorder class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value) extends TestCpg { @@ -16,7 +17,9 @@ class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new JavascriptProcessor( ruleCache, @@ -28,11 +31,46 @@ class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies + ) + } +} + +class TestCpgWithJavaScriptBase(val fileSuffix: String, val language: Language.Value) extends TestCpg { + protected def getLanguageProcessor( + ruleCache: RuleCache, + privadoInput: PrivadoInput, + dataFlowCache: DataFlowCache, + auditCache: AuditCache, + s3DatabaseDetailsCache: S3DatabaseDetailsCache, + appCache: AppCache, + propertyFilterCache: PropertyFilterCache, + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencyInfo: List[DependencyInfo] + ): BaseProcessor = { + new JavascriptBaseCPGProcessor( + ruleCache, + privadoInput.copy(fileLinkingReport = true, isDeltaFileScan = true), + privadoInput.sourceLocation.head, + dataFlowCache, + auditCache, + s3DatabaseDetailsCache, + appCache, + StatsRecorder(), + returnClosedCpg = false, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata ) } } class JavaScriptFrontendTestSuite(fileSuffix: String = ".js", language: Language.Value = Language.JAVASCRIPT) extends PrivadoBaseTestFixture(() => new TestCpgWithJavaScript(fileSuffix, language)) {} + +class JavaScriptBaseCpgFrontendTestSuite(fileSuffix: String = ".js", language: Language.Value = Language.JAVASCRIPT) + extends PrivadoBaseTestFixture(() => new TestCpgWithJavaScriptBase(fileSuffix, language)) {} diff --git a/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala index cbae891ff..63ed2d330 100644 --- a/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.kotlin.processor.KotlinProcessor import ai.privado.model.Language @@ -24,7 +17,9 @@ class TestCpgWithKotlin(val fileSuffix: String, val language: Language.Value) ex s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new KotlinProcessor( ruleCache, @@ -36,8 +31,10 @@ class TestCpgWithKotlin(val fileSuffix: String, val language: Language.Value) ex appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala b/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala index 0d23cc45a..16d3651ee 100644 --- a/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala +++ b/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala @@ -2,6 +2,7 @@ package ai.privado.testfixtures import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.model.Language import ai.privado.rule.RuleInfoTestData @@ -21,6 +22,8 @@ trait LanguageFrontend { private var appCache: Option[AppCache] = None private var propertyFilterCache: Option[PropertyFilterCache] = None private var databaseDetailsCache: Option[DatabaseDetailsCache] = None + private var fileLinkingMetadata: Option[FileLinkingMetadata] = None + private var dependencies: Option[List[DependencyInfo]] = None def setPrivadoInput(privadoInput: PrivadoInput): Unit = { if (this.privadoInput.isDefined) { @@ -78,6 +81,22 @@ trait LanguageFrontend { this.databaseDetailsCache = Some(databaseDetailsCache) } + def setFileLinkingMetadata(fileLinkingMetadata: FileLinkingMetadata): Unit = { + if (this.fileLinkingMetadata.isDefined) { + throw new RuntimeException("FileLinkingMetadata may only be set once per test") + } + this.fileLinkingMetadata = Some(fileLinkingMetadata) + } + + def getFileLinkingMetadata: FileLinkingMetadata = this.fileLinkingMetadata.getOrElse(FileLinkingMetadata()) + + def setDependencies(dependencies: List[DependencyInfo]): Unit = { + if (this.dependencies.isDefined) { + throw new RuntimeException("Dependencies may only be set once per test") + } + this.dependencies = Some(dependencies) + } + protected def getProcessor(sourceCodePath: java.io.File): BaseProcessor = { val privadoInput = this.privadoInput.getOrElse(PrivadoInput()).copy(sourceLocation = Set(sourceCodePath.getAbsolutePath)) @@ -93,7 +112,9 @@ trait LanguageFrontend { this.s3DatabaseDetailsCache.getOrElse(S3DatabaseDetailsCache()), appCache, this.propertyFilterCache.getOrElse(PropertyFilterCache()), - this.databaseDetailsCache.getOrElse(DatabaseDetailsCache()) + this.databaseDetailsCache.getOrElse(DatabaseDetailsCache()), + this.fileLinkingMetadata.getOrElse(FileLinkingMetadata()), + this.dependencies.getOrElse(List()) ) } @@ -105,6 +126,8 @@ trait LanguageFrontend { s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor } diff --git a/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala index 1cbf91a58..2000f0037 100644 --- a/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.php.processor.PhpProcessor import ai.privado.model.* @@ -24,7 +17,9 @@ class TestCpgWithPhp(val fileSuffix: String, val language: Language.Value) exten s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new PhpProcessor( ruleCache, @@ -36,8 +31,10 @@ class TestCpgWithPhp(val fileSuffix: String, val language: Language.Value) exten appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala index c347b3835..d8e150f6a 100644 --- a/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala @@ -1,20 +1,12 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.python.processor.PythonProcessor -import ai.privado.model.Language +import ai.privado.model.* import ai.privado.utility.StatsRecorder -import ai.privado.model.{CatLevelOne, Constants, FilterProperty, NodeType, RuleInfo} class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) extends TestCpg { @@ -26,7 +18,9 @@ class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) ex s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = new PythonProcessor( ruleCache, privadoInput, @@ -37,8 +31,10 @@ class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) ex appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala index 106a7a8e3..47ffb79ca 100644 --- a/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.ruby.processor.RubyProcessor import ai.privado.model.Language @@ -25,7 +18,9 @@ class TestCpgWithRuby(val fileSuffix: String, val language: Language.Value) exte s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencies: List[DependencyInfo] ): BaseProcessor = { new RubyProcessor( ruleCache, @@ -37,8 +32,10 @@ class TestCpgWithRuby(val fileSuffix: String, val language: Language.Value) exte appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/TestCpg.scala b/src/test/scala/ai/privado/testfixtures/TestCpg.scala index d60100113..b3335adc5 100644 --- a/src/test/scala/ai/privado/testfixtures/TestCpg.scala +++ b/src/test/scala/ai/privado/testfixtures/TestCpg.scala @@ -1,7 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{AppCache, AuditCache, DataFlowCache, PropertyFilterCache, RuleCache, S3DatabaseDetailsCache} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import io.circe.Json import io.shiftleft.codepropertygraph.Cpg import overflowdb.Graph @@ -50,11 +51,26 @@ abstract class TestCpg extends Cpg() with TestCodeWriter with LanguageFrontend { this } + def withFileLinkingMetadata(fileLinkingMetadata: FileLinkingMetadata): this.type = { + setFileLinkingMetadata(fileLinkingMetadata) + this + } + + def withDependencies(dependencies: List[DependencyInfo]): this.type = { + setDependencies(dependencies) + this + } + def getPrivadoJson() = { graph _privadoJson.get } + def getFileLinkingData: FileLinkingMetadata = { + graph + this.getFileLinkingMetadata + } + override def graph: Graph = { if (_graph.isEmpty) { val codeDir = writeCode(fileSuffix)