From df290127ef7d1e574122a594d5aae3f5a2add21b Mon Sep 17 00:00:00 2001 From: Pandurang Patil <5101898+pandurangpatil@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:46:26 +0530 Subject: [PATCH 1/2] Changes for surfacing dependency information under sink processing section. (#1261) * Code refactore Moved input configuration processing related code inside new package `inputprocessor`. * Json parser implementation along with unit test Json parser implementation along with unit test to parse Dependency information json passed through external config. * extend dependency node 1. Extend dependency node to add additional required properties and file edge. 2. Initial DependencyNode pass. TODO: Need to add unit test 3. Unfinished refactoring of PropertyPass. * dependency node creation and tagging for processing section. 1. Dependency node creation. 2. Dependency node tagging with dynamic rule id. 3. Required changes in export flow to consider dependency node for generating sink processing section. 4. Changes to take dependency information passed on through external rules passed and pass the deserialised DependencyInfo list to each languages processor. 5. Corresponding changes in each language processor to invoke commonly configured Dependency node creation and tagging passes generically. 6. Had to make corresponding changes in test suite in order to pass DependencyInfo list while writing the unit test. * Additional unit tests with multiple sinks * review comment fixes along with additional unit tests * review comment fixes --- schema/src/main/scala/CpgExtSchema.scala | 13 +- .../ai/privado/entrypoint/CommandParser.scala | 2 +- .../entrypoint/MetadataProcessor.scala | 16 +- ...tor.scala => RuleValidatorProcessor.scala} | 2 +- .../ai/privado/entrypoint/ScanProcessor.scala | 40 +- .../ai/privado/exporter/SinkExporter.scala | 2 +- .../DependencyTaggingProcessor.scala | 43 ++ .../DynamicRuleMerger.scala | 5 +- .../RuleProcessor.scala | 9 +- .../base/processor/BaseProcessor.scala | 23 +- .../csharp/processor/CSharpProcessor.scala | 33 +- .../go/processor/GoProcessor.scala | 21 +- .../java/processor/JavaProcessor.scala | 34 +- .../JavascriptBaseCPGProcessor.scala | 4 +- .../processor/JavascriptProcessor.scala | 38 +- .../kotlin/processor/KotlinProcessor.scala | 50 +- .../php/processor/PhpProcessor.scala | 20 +- .../processor/PythonBaseCPGProcessor.scala | 15 +- .../python/processor/PythonProcessor.scala | 17 +- .../ruby/processor/RubyProcessor.scala | 43 +- .../privado/passes/DependencyNodePass.scala | 43 ++ .../privado/passes/PropertyParserPass.scala | 49 +- .../scala/ai/privado/passes/Utility.scala | 21 + .../tagger/sink/DependencyNodeTagger.scala | 34 ++ .../exporter/PropertyFilterExportTest.scala | 2 +- .../RepoConfigMetadataExporterTest.scala | 2 +- .../DependencyTaggingProcessorTest.scala | 60 +++ .../DynamicRuleMergerTest.scala | 2 +- .../passes/config/GoYamlLinkerPassTest.scala | 17 +- .../config/JavaYamlLinkerPassTest.scala | 5 +- .../config/PropertiesFilePassTest.scala | 11 +- .../config/PropertyPassFilterTest.scala | 6 +- .../tagger/sink/DependencyTaggerTest.scala | 459 ++++++++++++++++++ .../config/JSPropertiesFilePassTest.scala | 8 +- .../RubyEnvPropertiesFilePassTest.scala | 6 +- .../RubyPropertiesFilePassTestBase.scala | 2 +- .../RubyYamlPropertiesFilePassTest.scala | 6 +- .../ruby/monolith/MonolithTest.scala | 14 +- .../schema/RubyMongoSchemaMapperTest.scala | 14 +- .../passes/SQLPropertyParserTest.scala | 10 +- .../testfixtures/CFrontendTestSuite.scala | 14 +- .../CSharpFrontendTestSuite.scala | 17 +- .../DefaultFrontendTestSuite.scala | 14 +- .../testfixtures/GoFrontendTestSuite.scala | 19 +- .../testfixtures/JavaFrontendTestSuite.scala | 9 +- .../JavaScriptFrontendTestSuite.scala | 9 +- .../KotlinFrontendTestSuite.scala | 17 +- .../testfixtures/LanguageFrontend.scala | 15 +- .../testfixtures/PhpFrontendTestSuite.scala | 17 +- .../PythonFrontendTestSuite.scala | 20 +- .../testfixtures/RubyFrontendTestSuite.scala | 17 +- .../ai/privado/testfixtures/TestCpg.scala | 8 +- 52 files changed, 969 insertions(+), 408 deletions(-) rename src/main/scala/ai/privado/entrypoint/{RuleValidator.scala => RuleValidatorProcessor.scala} (98%) create mode 100644 src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala rename src/main/scala/ai/privado/{entrypoint => inputprocessor}/DynamicRuleMerger.scala (94%) rename src/main/scala/ai/privado/{entrypoint => inputprocessor}/RuleProcessor.scala (99%) create mode 100644 src/main/scala/ai/privado/passes/DependencyNodePass.scala create mode 100644 src/main/scala/ai/privado/passes/Utility.scala create mode 100644 src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala create mode 100644 src/test/scala/ai/privado/inputprocessor/DependencyTaggingProcessorTest.scala rename src/test/scala/ai/privado/{entrypoint => inputprocessor}/DynamicRuleMergerTest.scala (99%) create mode 100644 src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala diff --git a/schema/src/main/scala/CpgExtSchema.scala b/schema/src/main/scala/CpgExtSchema.scala index 2550516b3..0807e1207 100644 --- a/schema/src/main/scala/CpgExtSchema.scala +++ b/schema/src/main/scala/CpgExtSchema.scala @@ -147,7 +147,7 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) { .addProperty(artifactId) .addProperty(configVersion) - val dependency = builder + val moduleDependency = builder .addNodeType(CpgSchemaConstants.MODULE_DEPENDENCY_NODE_NAME) .addProperty(groupId) .addProperty(artifactId) @@ -159,11 +159,17 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) { val dependencyModuleEdge = builder .addEdgeType(CpgSchemaConstants.DEPENDENCY_MODULE_EDGE_NAME) - module.addOutEdge(edge = dependencies, inNode = dependency) + module.addOutEdge(edge = dependencies, inNode = moduleDependency) module.addOutEdge(edge = sourceFile, inNode = file) + moduleDependency.addOutEdge(edge = sourceFile, inNode = file) + moduleDependency.addOutEdge(edge = dependencyModuleEdge, inNode = module) + + // Extend Dependency Node dependency.addOutEdge(edge = sourceFile, inNode = file) - dependency.addOutEdge(edge = dependencyModuleEdge, inNode = module) + dependency.addOutEdge(edge = taggedBy, inNode = tag) + dependency.extendz(astNode) + // Extend teamplateDOM node templateDOM.addOutEdge(edge = sourceFile, inNode = file) // Nodes and edges for Android res/layout/*.xml files and @@ -228,6 +234,7 @@ class CpgExtSchema(builder: SchemaBuilder, cpgSchema: CpgSchema) { private val derivedSourceEdge = builder.addEdgeType(CpgSchemaConstants.DERIVED_SOURCE_EDGE_NAME) astNode.addOutEdge(edge = originalSourceEdge, inNode = astNode) astNode.addOutEdge(edge = derivedSourceEdge, inNode = astNode) + } object CpgExtSchema { diff --git a/src/main/scala/ai/privado/entrypoint/CommandParser.scala b/src/main/scala/ai/privado/entrypoint/CommandParser.scala index db7d4c76a..0cebbf3c3 100644 --- a/src/main/scala/ai/privado/entrypoint/CommandParser.scala +++ b/src/main/scala/ai/privado/entrypoint/CommandParser.scala @@ -160,7 +160,7 @@ object CommandParser { Map( CommandConstants.SCAN -> ScanProcessor, CommandConstants.UPLOAD -> UploadProcessor, - CommandConstants.VALIDATE -> RuleValidator, + CommandConstants.VALIDATE -> RuleValidatorProcessor, CommandConstants.METADATA -> MetadataProcessor ) def parse(args: Array[String], statsRecorder: StatsRecorder): Option[CommandProcessor] = { diff --git a/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala b/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala index 9faaa260f..a6b1c2ead 100644 --- a/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala +++ b/src/main/scala/ai/privado/entrypoint/MetadataProcessor.scala @@ -1,25 +1,17 @@ package ai.privado.entrypoint -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - FileLinkingMetadata, - PropertyFilterCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* +import ai.privado.entrypoint.MetadataProcessor.statsRecorder +import ai.privado.inputprocessor.RuleProcessor import ai.privado.languageEngine.javascript.processor.JavascriptBaseCPGProcessor import ai.privado.languageEngine.python.processor.PythonBaseCPGProcessor import ai.privado.metadata.SystemInfo import ai.privado.metric.MetricHandler -import ai.privado.model.Constants -import ai.privado.model.Language.UNKNOWN import ai.privado.model.* +import ai.privado.model.Language.UNKNOWN import better.files.File import io.circe.Json import io.joern.console.cpgcreation.guessLanguage -import ai.privado.entrypoint.MetadataProcessor.statsRecorder import scala.util.{Failure, Success, Try} diff --git a/src/main/scala/ai/privado/entrypoint/RuleValidator.scala b/src/main/scala/ai/privado/entrypoint/RuleValidatorProcessor.scala similarity index 98% rename from src/main/scala/ai/privado/entrypoint/RuleValidator.scala rename to src/main/scala/ai/privado/entrypoint/RuleValidatorProcessor.scala index f997233ef..78834b8f3 100644 --- a/src/main/scala/ai/privado/entrypoint/RuleValidator.scala +++ b/src/main/scala/ai/privado/entrypoint/RuleValidatorProcessor.scala @@ -6,7 +6,7 @@ import ai.privado.rulevalidator.YamlFileValidator import better.files.File import org.slf4j.LoggerFactory -object RuleValidator extends CommandProcessor { +object RuleValidatorProcessor extends CommandProcessor { private val logger = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala b/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala index 99fcdfe4d..8b05a36cb 100644 --- a/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala +++ b/src/main/scala/ai/privado/entrypoint/ScanProcessor.scala @@ -24,6 +24,7 @@ package ai.privado.entrypoint import ai.privado.cache.* import ai.privado.entrypoint.ScanProcessor.statsRecorder +import ai.privado.inputprocessor.{DependencyInfo, DependencyTaggingProcessor, RuleProcessor} import ai.privado.languageEngine.c.processor.CProcessor import ai.privado.languageEngine.csharp.processor.CSharpProcessor import ai.privado.languageEngine.default.processor.DefaultProcessor @@ -37,7 +38,7 @@ import ai.privado.languageEngine.ruby.processor.RubyProcessor import ai.privado.metadata.SystemInfo import ai.privado.metric.MetricHandler import ai.privado.model.* -import ai.privado.model.Language.{Language, UNKNOWN} +import ai.privado.model.Language.UNKNOWN import ai.privado.utility.StatsRecorder import better.files.File import io.circe.Json @@ -47,9 +48,9 @@ import org.slf4j.LoggerFactory import privado_core.BuildInfo import scala.sys.exit -import scala.util.{Try} +import scala.util.Try -object ScanProcessor extends CommandProcessor with RuleProcessor { +object ScanProcessor extends CommandProcessor with RuleProcessor with DependencyTaggingProcessor { private val logger = LoggerFactory.getLogger(this.getClass) override def process(appCache: AppCache): Either[String, Unit] = { @@ -96,7 +97,11 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { config.forceLanguage } MetricHandler.metricsData("language") = Json.fromString(languageDetected.toString) - + val dependencies: List[DependencyInfo] = config.externalConfigPath.headOption match { + case Some(externalConfigPath) => + parseDependencyInfo(generateDependencyInfoJsonPath(externalConfigPath)) + case None => List() + } languageDetected match { case Language.JAVA => statsRecorder.justLogMessage("Detected language 'Java'") @@ -118,7 +123,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() else KotlinProcessor( @@ -132,7 +138,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.JAVASCRIPT => statsRecorder.justLogMessage("Detected language 'JavaScript'") @@ -147,7 +154,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.PYTHON => statsRecorder.justLogMessage("Detected language 'Python'") @@ -162,7 +170,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { propertyFilterCache = propertyFilterCache, databaseDetailsCache = databaseDetailsCache, statsRecorder = statsRecorder, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.RUBY => statsRecorder.justLogMessage("Detected language 'Ruby'") @@ -176,7 +185,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { appCache, propertyFilterCache = propertyFilterCache, statsRecorder = statsRecorder, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.GO => statsRecorder.justLogMessage("Detected language 'Go'") @@ -191,7 +201,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { propertyFilterCache = propertyFilterCache, statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.KOTLIN => statsRecorder.justLogMessage("Detected language 'Kotlin'") @@ -206,7 +217,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.CSHARP => statsRecorder.justLogMessage("Detected language 'C#'") @@ -221,7 +233,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ).processCpg() case Language.PHP => statsRecorder.justLogMessage("Detected language 'PHP'") @@ -236,7 +249,8 @@ object ScanProcessor extends CommandProcessor with RuleProcessor { statsRecorder = statsRecorder, databaseDetailsCache = databaseDetailsCache, propertyFilterCache = propertyFilterCache, - fileLinkingMetadata = fileLinkingMetadata + fileLinkingMetadata = fileLinkingMetadata, + dependencies = dependencies ) .processCpg() case Language.C => diff --git a/src/main/scala/ai/privado/exporter/SinkExporter.scala b/src/main/scala/ai/privado/exporter/SinkExporter.scala index 7b8e9e53c..f93c45bdd 100644 --- a/src/main/scala/ai/privado/exporter/SinkExporter.scala +++ b/src/main/scala/ai/privado/exporter/SinkExporter.scala @@ -145,7 +145,7 @@ class SinkExporter( .where(filterSink) .l ++ cpg.argument.isFieldIdentifier.where(filterSink).l ++ cpg.method.where(filterSink).l ++ cpg.dbNode .where(filterSink) - .l ++ cpg.highTouchSink.where(filterSink).l + .l ++ cpg.highTouchSink.where(filterSink).l ++ cpg.dependency.where(filterSink).l ExporterUtility.filterNodeBasedOnRepoItemTagName(sinks, repoItemTagName) } diff --git a/src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala b/src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala new file mode 100644 index 000000000..3dac778e4 --- /dev/null +++ b/src/main/scala/ai/privado/inputprocessor/DependencyTaggingProcessor.scala @@ -0,0 +1,43 @@ +package ai.privado.inputprocessor + +import org.slf4j.LoggerFactory +import upickle.default.* + +import java.io.File as JFile +import java.nio.file.{Files, Paths} +import scala.util.{Failure, Success, Try} +case class DependencyInfo( + groupId: String, + dependencyName: String, + version: String, + code: String, + ruleId: String, + ruleName: String, + ruleDomains: List[String], + ruleTags: List[String], + lineNumber: Int, + filePath: String +) { + def getFullDependencyName(): String = { + if groupId.isEmpty then dependencyName else s"$groupId.$dependencyName" + } +} + +object DependencyInfo { + implicit val reader: Reader[DependencyInfo] = macroR[DependencyInfo] +} +trait DependencyTaggingProcessor { + private val logger = LoggerFactory.getLogger(this.getClass) + + def generateDependencyInfoJsonPath(externalConfigPath: String): String = + List(externalConfigPath, "config", "dependencyInfo", "dependencyinfo.json").mkString(JFile.separator) + def parseDependencyInfo(filePath: String): List[DependencyInfo] = { + // Manually provide a reader for List[DependencyInfo] + Try(read[List[DependencyInfo]](new String(Files.readAllBytes(Paths.get(filePath))))) match { + case Success(dependencies) => dependencies + case Failure(exception) => + logger.error(s"Error while parsing $filePath : ", exception) + List() + } + } +} diff --git a/src/main/scala/ai/privado/entrypoint/DynamicRuleMerger.scala b/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala similarity index 94% rename from src/main/scala/ai/privado/entrypoint/DynamicRuleMerger.scala rename to src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala index 63a8fd0f4..3dfdfb14e 100644 --- a/src/main/scala/ai/privado/entrypoint/DynamicRuleMerger.scala +++ b/src/main/scala/ai/privado/inputprocessor/DynamicRuleMerger.scala @@ -1,11 +1,10 @@ -package ai.privado.entrypoint +package ai.privado.inputprocessor import ai.privado.model.{ConfigAndRules, FilterProperty, RuleInfo} import org.slf4j.LoggerFactory import scala.collection.mutable -import scala.collection.mutable.Map -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{ListBuffer, Map} trait DynamicRuleMerger { diff --git a/src/main/scala/ai/privado/entrypoint/RuleProcessor.scala b/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala similarity index 99% rename from src/main/scala/ai/privado/entrypoint/RuleProcessor.scala rename to src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala index b89cc68a2..54fb69ece 100644 --- a/src/main/scala/ai/privado/entrypoint/RuleProcessor.scala +++ b/src/main/scala/ai/privado/inputprocessor/RuleProcessor.scala @@ -1,19 +1,20 @@ -package ai.privado.entrypoint +package ai.privado.inputprocessor import ai.privado.cache.{AppCache, RuleCache} +import ai.privado.entrypoint.PrivadoInput import ai.privado.metric.MetricHandler import ai.privado.model.* import ai.privado.model.Language.Language import ai.privado.rulevalidator.YamlFileValidator import ai.privado.utility.StatsRecorder +import ai.privado.utility.Utilities.{isValidDEDRule, isValidRule} import better.files.File +import io.circe.Json +import io.circe.yaml.parser import org.slf4j.LoggerFactory import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable import scala.sys.exit -import io.circe.yaml.parser -import ai.privado.utility.Utilities.{isValidDEDRule, isValidRule} -import io.circe.Json trait RuleProcessor extends DynamicRuleMerger { diff --git a/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala b/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala index 07e33dd4c..e250ce8a8 100644 --- a/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala @@ -5,22 +5,20 @@ import ai.privado.cache.* import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.PrivadoInput import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.java.cache.ModuleCache -import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, ModuleFilePass} +import ai.privado.languageEngine.java.passes.config.ModuleFilePass import ai.privado.languageEngine.java.passes.module.{DependenciesCategoryPass, DependenciesNodePass} import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* import ai.privado.model.Language.Language import ai.privado.model.{CpgWithOutputMap, Language} -import ai.privado.passes.ExperimentalLambdaDataFlowSupportPass -import ai.privado.semantic.language.* -import ai.privado.tagger.PrivadoParallelCpgPass -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} +import ai.privado.passes.{DependencyNodePass, ExperimentalLambdaDataFlowSupportPass} +import ai.privado.tagger.sink.DependencyNodeTagger +import ai.privado.utility.{StatsRecorder, UnresolvedReportUtility} import io.circe.Json import io.joern.dataflowengineoss.language.Path import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} -import io.joern.javasrc2cpg.Config -import io.joern.x2cpg.X2CpgConfig import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.ModuleDependency import io.shiftleft.passes.CpgPassBase @@ -28,10 +26,10 @@ import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.{Logger, LoggerFactory} -import java.util.Calendar import scala.collection.mutable import scala.collection.mutable.ListBuffer import scala.util.{Failure, Success, Try} + abstract class BaseProcessor( ruleCache: RuleCache, privadoInput: PrivadoInput, @@ -45,7 +43,8 @@ abstract class BaseProcessor( returnClosedCpg: Boolean, databaseDetailsCache: DatabaseDetailsCache, propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] = List() ) { val logger: Logger = LoggerFactory.getLogger(getClass) @@ -102,7 +101,7 @@ abstract class BaseProcessor( * @param cpg * @return */ - def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List() + def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List(DependencyNodePass(cpg, dependencies, sourceRepoLocation)) /** Method to apply Dataflow pass * @param cpg @@ -154,7 +153,9 @@ abstract class BaseProcessor( result } - def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = ??? + def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + DependencyNodeTagger(cpg, dependencies, ruleCache).createAndApply() + } protected def applyFinalExport( cpg: Cpg, diff --git a/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala b/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala index 0a3279ce4..a328528e1 100644 --- a/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala @@ -23,40 +23,24 @@ package ai.privado.languageEngine.csharp.processor -import ai.privado.audit.AuditReportEntryPoint import ai.privado.cache.* -import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.* import ai.privado.entrypoint.ScanProcessor.config -import ai.privado.entrypoint.ScanProcessor -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.csharp.semantic.Language.tagger -import ai.privado.languageEngine.java.passes.config.JavaPropertyLinkerPass -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* -import ai.privado.model.Language.Language -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} -import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.model.{Constants, CpgWithOutputMap, Language} +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder} -import better.files.File -import io.circe.Json import io.joern.csharpsrc2cpg.* -import io.joern.dataflowengineoss.language.Path -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.x2cpg.X2Cpg import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.LoggerFactory -import java.util.Calendar -import scala.collection.mutable.ListBuffer -import scala.util.{Failure, Success, Try} +import scala.util.Try class CSharpProcessor( ruleCache: RuleCache, @@ -70,7 +54,8 @@ class CSharpProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -84,15 +69,17 @@ class CSharpProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { private val logger = LoggerFactory.getLogger(getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List[CpgPassBase]() + super.applyPrivadoPasses(cpg) } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala b/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala index 5785df6d1..888c4973d 100644 --- a/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala @@ -1,29 +1,25 @@ package ai.privado.languageEngine.go.processor -import ai.privado.audit.AuditReportEntryPoint import ai.privado.cache.* -import ai.privado.dataflow.Dataflow -import ai.privado.entrypoint.ScanProcessor.config import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.go.passes.SQLQueryParser import ai.privado.languageEngine.go.passes.config.GoYamlLinkerPass import ai.privado.languageEngine.go.passes.orm.ORMParserPass import ai.privado.languageEngine.go.semantic.Language.tagger -import ai.privado.semantic.* import ai.privado.model.Constants.* -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.semantic.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder} import io.joern.gosrc2cpg.{Config, GoSrc2Cpg} import io.joern.x2cpg.X2Cpg import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory class GoProcessor( @@ -38,7 +34,8 @@ class GoProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -52,12 +49,13 @@ class GoProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { private val logger = LoggerFactory.getLogger(getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List( + super.applyPrivadoPasses(cpg) ++ List( { if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") @@ -72,6 +70,7 @@ class GoProcessor( } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala index dea731054..2e4895aa1 100644 --- a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala @@ -23,42 +23,30 @@ package ai.privado.languageEngine.java.processor -import ai.privado.audit.{AuditReportEntryPoint, DependencyReport} import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor -import ai.privado.languageEngine.java.cache.ModuleCache -import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, JavaYamlLinkerPass, ModuleFilePass} +import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, JavaYamlLinkerPass} import ai.privado.languageEngine.java.passes.methodFullName.LoggerLombokPass -import ai.privado.languageEngine.java.passes.module.{DependenciesCategoryPass, DependenciesNodePass} import ai.privado.languageEngine.java.semantic.Language.* -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* -import ai.privado.model.Language.Language -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* import ai.privado.tagger.PrivadoParallelCpgPass +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} import better.files.File -import io.circe.Json -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.joern.x2cpg.utils.ExternalCommand import io.joern.x2cpg.utils.dependency.DependencyResolver +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.JavaProperty -import io.shiftleft.codepropertygraph.generated.{Cpg, Languages} import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.{Logger, LoggerFactory} import java.nio.file.Paths -import java.util.Calendar -import scala.collection.mutable.ListBuffer import scala.util.{Failure, Success, Try} class JavaProcessor( @@ -73,7 +61,8 @@ class JavaProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -87,14 +76,15 @@ class JavaProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger: Logger = LoggerFactory.getLogger(getClass) private var cpgconfig = Config() override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List({ + super.applyPrivadoPasses(cpg) ++ List({ if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") else @@ -110,7 +100,8 @@ class JavaProcessor( ) } - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -122,6 +113,7 @@ class JavaProcessor( statsRecorder, fileLinkingMetadata ) + } override def processCpg(): Either[String, CpgWithOutputMap] = { val excludeFileRegex = ruleCache.getExclusionRegex diff --git a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala index 0a5cd342e..503d3dc65 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptBaseCPGProcessor.scala @@ -12,6 +12,7 @@ import ai.privado.cache.{ TaggerCache } import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.metric.MetricHandler import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} import ai.privado.model.CpgWithOutputMap @@ -49,7 +50,8 @@ class JavascriptBaseCPGProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + List() ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala index 6735357cb..d8a584fb0 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala @@ -23,38 +23,25 @@ package ai.privado.languageEngine.javascript.processor -import ai.privado.audit.AuditReportEntryPoint import ai.privado.cache.* -import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.PrivadoInput -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.javascript.metadata.FileImportMappingPassJS import ai.privado.languageEngine.javascript.passes.config.{JSPropertyLinkerPass, JsConfigPropertyPass} import ai.privado.languageEngine.javascript.semantic.Language.* -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} -import better.files.File import io.joern.jssrc2cpg.{Config, JsSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays -import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.shiftleft.codepropertygraph -import io.shiftleft.codepropertygraph.generated.{Cpg, Operators} +import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} -import java.nio.file.Paths -import java.util.Calendar -import scala.collection.mutable.ListBuffer -import scala.jdk.CollectionConverters.CollectionHasAsScala -import scala.util.{Failure, Success, Try} - class JavascriptProcessor( ruleCache: RuleCache, privadoInput: PrivadoInput, @@ -67,7 +54,8 @@ class JavascriptProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -81,15 +69,16 @@ class JavascriptProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - val passesList = List(new HTMLParserPass(cpg, sourceRepoLocation, ruleCache, privadoInputConfig = privadoInput)) - - passesList ++ List({ + super.applyPrivadoPasses(cpg) ++ List( + new HTMLParserPass(cpg, sourceRepoLocation, ruleCache, privadoInputConfig = privadoInput) + ) ++ List({ if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") new JsConfigPropertyPass(cpg) @@ -110,7 +99,8 @@ class JavascriptProcessor( ) } - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -121,7 +111,7 @@ class JavascriptProcessor( statsRecorder, fileLinkingMetadata ) - + } override def applyDataflowAndPostProcessingPasses(cpg: Cpg): Unit = { super.applyDataflowAndPostProcessingPasses(cpg) if (privadoInput.disablePostProcessingPass) diff --git a/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala b/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala index e0bd2f0ad..708fb1850 100644 --- a/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/kotlin/processor/KotlinProcessor.scala @@ -1,50 +1,22 @@ package ai.privado.languageEngine.kotlin.processor -import ai.privado.audit.{AuditReportEntryPoint, DependencyReport} -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - FileLinkingMetadata, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache, - TaggerCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput -import ai.privado.exporter.{ExcelExporter, JSONExporter} +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor -import ai.privado.languageEngine.java.cache.ModuleCache -import ai.privado.languageEngine.java.passes.config.{JavaPropertyLinkerPass, ModuleFilePass} -import ai.privado.languageEngine.java.passes.module.{DependenciesCategoryPass, DependenciesNodePass} +import ai.privado.languageEngine.java.passes.config.JavaPropertyLinkerPass import ai.privado.languageEngine.kotlin.semantic.Language.* -import ai.privado.metric.MetricHandler import ai.privado.model.Constants.* -import ai.privado.model.Language.Language -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} -import better.files.File -import io.circe.Json -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.kotlin2cpg.{Config, Kotlin2Cpg} import io.joern.x2cpg.X2Cpg -import io.joern.x2cpg.passes.base.AstLinkerPass -import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* -import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.slf4j.LoggerFactory - -import java.nio.file.Paths -import java.util.Calendar -import scala.collection.mutable.ListBuffer -import scala.util.{Failure, Success, Try} class KotlinProcessor( ruleCache: RuleCache, privadoInput: PrivadoInput, @@ -57,7 +29,8 @@ class KotlinProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -71,13 +44,14 @@ class KotlinProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger = LoggerFactory.getLogger(getClass) private var cpgconfig = Config() override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List({ + super.applyPrivadoPasses(cpg) ++ List({ if (privadoInput.assetDiscovery) new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") else @@ -99,7 +73,8 @@ class KotlinProcessor( statsRecorder.endLastStage() } - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -110,6 +85,7 @@ class KotlinProcessor( statsRecorder, fileLinkingMetadata ) + } override def processCpg(): Either[String, CpgWithOutputMap] = { diff --git a/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala b/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala index 35b27be06..49d08fb4e 100644 --- a/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/php/processor/PhpProcessor.scala @@ -24,19 +24,16 @@ package ai.privado.languageEngine.php.processor import ai.privado.cache.* -import ai.privado.entrypoint.ScanProcessor.config import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.php.semantic.Language.tagger import ai.privado.model.Constants.* import ai.privado.model.{CpgWithOutputMap, Language} -import ai.privado.model.Language.Language -import ai.privado.model.{CpgWithOutputMap, Language} import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import io.circe.Json import io.joern.php2cpg.{Config, Php2Cpg} -import io.joern.x2cpg.X2Cpg.{applyDefaultOverlays, newEmptyCpg} +import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase import org.slf4j.{Logger, LoggerFactory} @@ -44,7 +41,6 @@ import org.slf4j.{Logger, LoggerFactory} import java.io.File import java.nio.file.Paths import java.util.Calendar -import scala.util.Try class PhpProcessor( ruleCache: RuleCache, @@ -58,7 +54,8 @@ class PhpProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -72,14 +69,16 @@ class PhpProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) - override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = List[CpgPassBase]() + override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = super.applyPrivadoPasses(cpg) - override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = + override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, @@ -90,6 +89,7 @@ class PhpProcessor( statsRecorder, fileLinkingMetadata ) + } override def applyDataflowAndPostProcessingPasses(cpg: Cpg): Unit = { super.applyDataflowAndPostProcessingPasses(cpg) diff --git a/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala b/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala index 10a68102d..a659b689e 100644 --- a/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/python/processor/PythonBaseCPGProcessor.scala @@ -1,16 +1,6 @@ package ai.privado.languageEngine.python.processor -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - FileLinkingMetadata, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache, - TaggerCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput import ai.privado.metric.MetricHandler import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} @@ -49,7 +39,8 @@ class PythonBaseCPGProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + List() ) { override val logger: Logger = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala b/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala index 880fbd7be..ff8b85c9e 100644 --- a/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/python/processor/PythonProcessor.scala @@ -1,20 +1,19 @@ package ai.privado.languageEngine.python.processor -import ai.privado.entrypoint.PrivadoInput import ai.privado.cache.* +import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.python.config.PythonConfigPropertyPass import ai.privado.languageEngine.python.metadata.FileLinkingMetadataPassPython import ai.privado.languageEngine.python.passes.PrivadoPythonTypeHintCallLinker import ai.privado.languageEngine.python.passes.config.PythonPropertyLinkerPass import ai.privado.languageEngine.python.semantic.Language.* -import ai.privado.languageEngine.python.tagger.PythonS3Tagger import ai.privado.model.Constants.* import ai.privado.model.{Constants, CpgWithOutputMap, Language} import ai.privado.passes.* -import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder} import better.files.File import io.joern.pysrc2cpg.* import io.joern.x2cpg.X2Cpg.applyDefaultOverlays @@ -23,7 +22,6 @@ import io.joern.x2cpg.passes.callgraph.NaiveCallLinker import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.passes.CpgPassBase -import io.shiftleft.semanticcpg.language.* import org.slf4j.LoggerFactory import java.nio.file.Paths @@ -40,7 +38,8 @@ class PythonProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -54,13 +53,14 @@ class PythonProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override val logger = LoggerFactory.getLogger(getClass) override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { - List( + super.applyPrivadoPasses(cpg) ++ List( new HTMLParserPass(cpg, sourceRepoLocation, ruleCache, privadoInputConfig = privadoInput), { if (privadoInput.assetDiscovery) { new JsonPropertyParserPass(cpg, s"$sourceRepoLocation/${Constants.generatedConfigFolderName}") @@ -77,6 +77,7 @@ class PythonProcessor( } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala b/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala index b4b452b35..bb2b7b82b 100644 --- a/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/ruby/processor/RubyProcessor.scala @@ -23,26 +23,20 @@ package ai.privado.languageEngine.ruby.processor -import ai.privado.audit.{AuditReportEntryPoint, DEDSourceDiscovery} import ai.privado.cache.* -import ai.privado.entrypoint.ScanProcessor.config -import ai.privado.entrypoint.{PrivadoInput, ScanProcessor} -import ai.privado.exporter.{ExcelExporter, JSONExporter} -import ai.privado.exporter.monolith.MonolithExporter +import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.ruby.passes.* import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.languageEngine.ruby.passes.download.DownloadDependenciesPass -import ai.privado.languageEngine.ruby.passes.* import ai.privado.languageEngine.ruby.semantic.Language.* -import ai.privado.metric.MetricHandler -import ai.privado.model.Constants.{cpgOutputFileName, outputAuditFileName, outputDirectoryName, outputFileName} -import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language} -import ai.privado.passes.{DBTParserPass, ExperimentalLambdaDataFlowSupportPass, JsonPropertyParserPass, SQLParser} -import ai.privado.semantic.language.* +import ai.privado.model.Constants.{cpgOutputFileName, outputDirectoryName} +import ai.privado.model.{Constants, CpgWithOutputMap, Language} +import ai.privado.passes.* +import ai.privado.utility.StatsRecorder import ai.privado.utility.Utilities.createCpgFolder -import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility} import better.files.File -import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.rubysrc2cpg.deprecated.astcreation.ResourceManagedParser import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.* @@ -58,28 +52,21 @@ import io.joern.x2cpg.passes.controlflow.cfgcreation.{Cfg, CfgCreator} import io.joern.x2cpg.passes.controlflow.cfgdominator.CfgDominatorPass import io.joern.x2cpg.passes.controlflow.codepencegraph.CdgPass import io.joern.x2cpg.passes.frontend.* -import io.joern.x2cpg.utils.ConcurrentTaskUtil import io.joern.x2cpg.{SourceFiles, ValidationMode, X2Cpg, X2CpgConfig} import io.shiftleft.codepropertygraph import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.codepropertygraph.generated.{Cpg, Languages, Operators} +import io.shiftleft.codepropertygraph.generated.{Cpg, Languages} +import io.shiftleft.passes.CpgPassBase import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext} -import org.slf4j.LoggerFactory import overflowdb.BatchedUpdate.DiffGraphBuilder -import ai.privado.dataflow.Dataflow -import ai.privado.cache.* -import ai.privado.languageEngine.base.processor.BaseProcessor -import io.shiftleft.passes.CpgPassBase import java.util -import java.util.Calendar -import java.util.concurrent.{Callable, Executors} import scala.collection.mutable.ListBuffer import scala.concurrent.* +import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.DurationLong import scala.util.{Failure, Success, Try, Using} -import scala.concurrent.ExecutionContext.Implicits.global class RubyProcessor( ruleCache: RuleCache, @@ -93,7 +80,8 @@ class RubyProcessor( returnClosedCpg: Boolean = true, databaseDetailsCache: DatabaseDetailsCache = new DatabaseDetailsCache(), propertyFilterCache: PropertyFilterCache = new PropertyFilterCache(), - fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata() + fileLinkingMetadata: FileLinkingMetadata = new FileLinkingMetadata(), + dependencies: List[DependencyInfo] ) extends BaseProcessor( ruleCache, privadoInput, @@ -107,7 +95,8 @@ class RubyProcessor( returnClosedCpg, databaseDetailsCache, propertyFilterCache, - fileLinkingMetadata + fileLinkingMetadata, + dependencies ) { override def applyPrivadoPasses(cpg: Cpg): List[CpgPassBase] = { @@ -124,7 +113,7 @@ class RubyProcessor( // Using our own pass by overriding languageEngine's pass // new RubyImportResolverPass(cpg, packageTableInfo).createAndApply() val globalSymbolTable = new SymbolTable[LocalKey](SBKey.fromNodeToLocalKey) - passesList ++ List(new GlobalImportPass(cpg, globalSymbolTable)) ++ + super.applyPrivadoPasses(cpg) ++ passesList ++ List(new GlobalImportPass(cpg, globalSymbolTable)) ++ new PrivadoRubyTypeRecoveryPassGenerator(cpg, globalSymbolTable).generate() ++ List( new RubyTypeHintCallLinker(cpg), @@ -133,10 +122,10 @@ class RubyProcessor( new SQLParser(cpg, sourceRepoLocation, ruleCache), new DBTParserPass(cpg, sourceRepoLocation, ruleCache, databaseDetailsCache) ) - passesList } override def runPrivadoTagger(cpg: Cpg, taggerCache: TaggerCache): Unit = { + super.runPrivadoTagger(cpg, taggerCache) cpg.runTagger( ruleCache, taggerCache, diff --git a/src/main/scala/ai/privado/passes/DependencyNodePass.scala b/src/main/scala/ai/privado/passes/DependencyNodePass.scala new file mode 100644 index 000000000..c634786ed --- /dev/null +++ b/src/main/scala/ai/privado/passes/DependencyNodePass.scala @@ -0,0 +1,43 @@ +package ai.privado.passes + +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.tagger.PrivadoSimpleCpgPass +import ai.privado.utility.Utilities +import io.shiftleft.codepropertygraph.generated.nodes.{NewDependency, NewFile} +import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes} + +import scala.collection.mutable + +/** This pass is using single threaded CPG pass mechanism. + * a. The contents to be processed are not going to be very huge as the list contains only the identified 3p + * dependencies only. b. We need to create a file node. Which can be repeated across the multiple nodes. If we use + * parallel mechanism. It will be difficult to handle that use case. + * + * @param cpg + * @param dependencies + * @param projectRoot + */ +class DependencyNodePass(cpg: Cpg, dependencies: List[DependencyInfo], projectRoot: String) + extends PrivadoSimpleCpgPass(cpg) + with Utility { + val fileNodeMap: mutable.Map[String, NewFile] = mutable.Map[String, NewFile]() + def run(builder: DiffGraphBuilder): Unit = { + dependencies.foreach(dependency => { + val dep = NewDependency() + .name(dependency.getFullDependencyName()) + .version(dependency.version) + .lineNumber(dependency.lineNumber) + .code(dependency.code) + + val fileNode = fileNodeMap.get(dependency.filePath) match { + case Some(fileNode) => fileNode + case None => + val fileNode = Utilities.addFileNode(dependency.filePath, builder) + fileNodeMap.put(dependency.filePath, fileNode) + fileNode + } + builder.addNode(dep) + builder.addEdge(dep, fileNode, EdgeTypes.SOURCE_FILE) + }) + } +} diff --git a/src/main/scala/ai/privado/passes/PropertyParserPass.scala b/src/main/scala/ai/privado/passes/PropertyParserPass.scala index eaa491744..eed01dbd1 100644 --- a/src/main/scala/ai/privado/passes/PropertyParserPass.scala +++ b/src/main/scala/ai/privado/passes/PropertyParserPass.scala @@ -1,4 +1,4 @@ -package ai.privado.utility +package ai.privado.passes import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.NewJavaProperty @@ -30,13 +30,14 @@ import org.yaml.snakeyaml.nodes.{MappingNode, Node, NodeTuple, ScalarNode, Seque import scala.jdk.CollectionConverters.* import ai.privado.model.{Constants, Language} import ai.privado.tagger.PrivadoParallelCpgPass +import ai.privado.utility.{ConfigParserUtility, Utilities} import org.yaml.snakeyaml.constructor.SafeConstructor import better.files.File.VisitOptions import java.nio.file.Path import scala.collection.mutable.ListBuffer -object FileExtensions { +object PropertyFileExtensions { val PROPERTIES = ".properties" val YAML = ".yaml" val YML = ".yml" @@ -54,7 +55,8 @@ class PropertyParserPass( language: Language.Value, propertyFilterCache: PropertyFilterCache = PropertyFilterCache(), privadoInput: PrivadoInput = PrivadoInput() -) extends PrivadoParallelCpgPass[String](cpg) { +) extends PrivadoParallelCpgPass[String](cpg) + with Utility { val PLACEHOLDER_TOKEN_START_END = "@@" val logger = LoggerFactory.getLogger(getClass) @@ -64,11 +66,11 @@ class PropertyParserPass( configFiles( projectRoot, Set( - FileExtensions.PROPERTIES, - FileExtensions.YAML, - FileExtensions.YML, - FileExtensions.XML, - FileExtensions.CONF + PropertyFileExtensions.PROPERTIES, + PropertyFileExtensions.YAML, + PropertyFileExtensions.YML, + PropertyFileExtensions.XML, + PropertyFileExtensions.CONF ) ).toArray } @@ -76,30 +78,40 @@ class PropertyParserPass( if (privadoInput.enableIngressAndEgressUrls) { configFiles( projectRoot, - Set(FileExtensions.JSON, FileExtensions.ENV, FileExtensions.YAML, FileExtensions.YML) + Set( + PropertyFileExtensions.JSON, + PropertyFileExtensions.ENV, + PropertyFileExtensions.YAML, + PropertyFileExtensions.YML + ) ).toArray } else { - configFiles(projectRoot, Set(FileExtensions.JSON, FileExtensions.ENV)).toArray + configFiles(projectRoot, Set(PropertyFileExtensions.JSON, PropertyFileExtensions.ENV)).toArray } case Language.PYTHON => configFiles( projectRoot, - Set(FileExtensions.INI, FileExtensions.ENV, FileExtensions.YAML, FileExtensions.YML) + Set( + PropertyFileExtensions.INI, + PropertyFileExtensions.ENV, + PropertyFileExtensions.YAML, + PropertyFileExtensions.YML + ) ).toArray case Language.RUBY => - (configFiles(projectRoot, Set(FileExtensions.ENV)) ++ configFiles( + (configFiles(projectRoot, Set(PropertyFileExtensions.ENV)) ++ configFiles( projectRoot, - Set(FileExtensions.YML, FileExtensions.YAML) + Set(PropertyFileExtensions.YML, PropertyFileExtensions.YAML) ).filter(_.matches(".*(settings|config).*"))).toArray // Ruby has a lot of yaml files so creating property nodes for all of them, exposes a lot of property nodes, // which are incorrect, so we go by the approach of being selective and creating property nodes for only the impacted files case Language.GO => - (configFiles(projectRoot, Set(FileExtensions.YAML, FileExtensions.YML))).toArray + (configFiles(projectRoot, Set(PropertyFileExtensions.YAML, PropertyFileExtensions.YML))).toArray } } override def runOnPart(builder: DiffGraphBuilder, file: String): Unit = { - val fileNode = addFileNode(file, builder) + val fileNode = addFileNode(file, builder, projectRoot) val propertyNodes = obtainKeyValuePairs(file, builder).map(pair => addPropertyNode(pair, builder)) propertyNodes.foreach(builder.addEdge(_, fileNode, EdgeTypes.SOURCE_FILE)) } @@ -354,13 +366,6 @@ class PropertyParserPass( propertyNode } - private def addFileNode(name: String, builder: BatchedUpdate.DiffGraphBuilder): NewFile = { - val relativeFileName = Path.of(projectRoot).relativize(Path.of(name)).toString - val fileNode = NewFile().name(relativeFileName) - builder.addNode(fileNode) - fileNode - } - private def configFiles(projectRoot: String, extensions: Set[String]): List[String] = { def getListOfFiles(dir: String): List[File] = { val d = new File(dir) diff --git a/src/main/scala/ai/privado/passes/Utility.scala b/src/main/scala/ai/privado/passes/Utility.scala new file mode 100644 index 000000000..99151007b --- /dev/null +++ b/src/main/scala/ai/privado/passes/Utility.scala @@ -0,0 +1,21 @@ +package ai.privado.passes + +import io.shiftleft.codepropertygraph.generated.nodes.NewFile +import overflowdb.BatchedUpdate + +import java.nio.file.Path + +trait Utility { + + def addFileNode(name: String, builder: BatchedUpdate.DiffGraphBuilder, projectRoot: String): NewFile = { + val relativeFileName = Path.of(projectRoot).relativize(Path.of(name)).toString + val fileNode = NewFile().name(relativeFileName) + builder.addNode(fileNode) + fileNode + } + + def combinedRulePattern(patterns: List[String]): String = { + patterns.mkString("(", "|", ")") + } + +} diff --git a/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala b/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala new file mode 100644 index 000000000..9c99072dd --- /dev/null +++ b/src/main/scala/ai/privado/tagger/sink/DependencyNodeTagger.scala @@ -0,0 +1,34 @@ +package ai.privado.tagger.sink + +import ai.privado.cache.RuleCache +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.passes.Utility +import ai.privado.tagger.PrivadoSimpleCpgPass +import ai.privado.utility.Utilities +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* +import org.slf4j.LoggerFactory + +class DependencyNodeTagger(cpg: Cpg, dependencies: List[DependencyInfo], ruleCache: RuleCache) + extends PrivadoSimpleCpgPass(cpg) + with Utility { + private val logger = LoggerFactory.getLogger(this.getClass) + def run(builder: DiffGraphBuilder): Unit = { + dependencies.foreach(dependency => { + cpg.dependency + .nameExact(dependency.getFullDependencyName()) + .where(_.file.nameExact(dependency.filePath)) + .headOption match { + case Some(dep) => + ruleCache.getRuleInfo(dependency.ruleId) match { + case Some(rule) => Utilities.addRuleTags(builder, dep, rule, ruleCache) + case None => + logger.error( + s"Dynamic rule ${dependency.ruleId} is not found inside rule cache. It seems given rule is not passed" + ) + } + case None => + } + }) + } +} diff --git a/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala b/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala index 89f94777e..07faaa135 100644 --- a/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala +++ b/src/test/scala/ai/privado/exporter/PropertyFilterExportTest.scala @@ -2,7 +2,7 @@ package ai.privado.exporter import ai.privado.cache.{PropertyFilterCache, RuleCache} import ai.privado.model.{ConfigAndRules, Language, SystemConfig} -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala b/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala index 9cbfdc29d..4ffede554 100644 --- a/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala +++ b/src/test/scala/ai/privado/exporter/RepoConfigMetadataExporterTest.scala @@ -2,7 +2,7 @@ package ai.privado.exporter import ai.privado.cache.RuleCache import ai.privado.model.{ConfigAndRules, Language, SystemConfig} -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/inputprocessor/DependencyTaggingProcessorTest.scala b/src/test/scala/ai/privado/inputprocessor/DependencyTaggingProcessorTest.scala new file mode 100644 index 000000000..504a8bbaa --- /dev/null +++ b/src/test/scala/ai/privado/inputprocessor/DependencyTaggingProcessorTest.scala @@ -0,0 +1,60 @@ +package ai.privado.inputprocessor + +import better.files.* +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class DependencyTaggingProcessorTest + extends AnyWordSpec + with Matchers + with BeforeAndAfterAll + with DependencyTaggingProcessor { + + "Parsing test" should { + "Working test" in { + File.usingTemporaryDirectory("deptest") { tmpDir => + val tempFile = tmpDir / "java.json"; + tempFile.writeText(""" + |[ + | { + | "groupId": "com.privado.sample", + | "dependencyName": "Scanner", + | "version": "1.0", + | "code" : "Scanner", + | "ruleId": "ThirdParties.SDK.Scanner", + | "ruleName": "Privado Scanner", + | "ruleDomains": [ + | "scanner.privado.ai" + | ], + | "ruleTags": [], + | "lineNumber": 10, + | "filePath": "/path/pom.xml" + | } + |] + |""".stripMargin) + val dependencies: List[DependencyInfo] = parseDependencyInfo(tempFile.pathAsString) + dependencies shouldBe List( + DependencyInfo( + "com.privado.sample", + "Scanner", + "1.0", + "Scanner", + "ThirdParties.SDK.Scanner", + "Privado Scanner", + List("scanner.privado.ai"), + List(), + 10, + "/path/pom.xml" + ) + ) + } + } + + "Graceful handling of error situation" in { + val dependencies: List[DependencyInfo] = parseDependencyInfo("path/nonexistingfile.json") + dependencies shouldBe List() + } + } + +} diff --git a/src/test/scala/ai/privado/entrypoint/DynamicRuleMergerTest.scala b/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala similarity index 99% rename from src/test/scala/ai/privado/entrypoint/DynamicRuleMergerTest.scala rename to src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala index b73cdbab3..7cf21b37e 100644 --- a/src/test/scala/ai/privado/entrypoint/DynamicRuleMergerTest.scala +++ b/src/test/scala/ai/privado/inputprocessor/DynamicRuleMergerTest.scala @@ -1,4 +1,4 @@ -package ai.privado.entrypoint +package ai.privado.inputprocessor import ai.privado.cache.RuleCache import ai.privado.model.{CatLevelOne, ConfigAndRules, Constants, FilterProperty, Language, NodeType, RuleInfo} diff --git a/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala b/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala index cf90fdd70..e68697e46 100644 --- a/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/go/passes/config/GoYamlLinkerPassTest.scala @@ -4,28 +4,19 @@ import ai.privado.cache.{AppCache, FileLinkingMetadata, RuleCache, TaggerCache} import ai.privado.entrypoint.PrivadoInput import ai.privado.languageEngine.go.tagger.sink.GoAPITagger import ai.privado.languageEngine.go.tagger.source.IdentifierTagger -import ai.privado.model.{ - CatLevelOne, - ConfigAndRules, - Constants, - FilterProperty, - Language, - NodeType, - RuleInfo, - SystemConfig -} -import ai.privado.utility.PropertyParserPass +import ai.privado.model.* +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.gosrc2cpg.{Config, GoSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import io.shiftleft.semanticcpg.language.* -import ai.privado.semantic.language.* abstract class GoYamlLinkerPassTest extends GoYamlFileLinkerPassTestBase { override val yamlFileContents: String = """ diff --git a/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala b/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala index af2482526..29e73e83a 100644 --- a/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/passes/config/JavaYamlLinkerPassTest.scala @@ -2,11 +2,12 @@ package ai.privado.languageEngine.java.passes.config import ai.privado.cache.{AppCache, FileLinkingMetadata, RuleCache, TaggerCache} import ai.privado.entrypoint.PrivadoInput -import ai.privado.semantic.language.* import ai.privado.languageEngine.java.tagger.sink.api.JavaAPITagger import ai.privado.languageEngine.java.tagger.source.* import ai.privado.model.* -import ai.privado.utility.{PropertyParserPass, StatsRecorder} +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* +import ai.privado.utility.StatsRecorder import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} diff --git a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala index 430cfc9a9..982f3f182 100644 --- a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertiesFilePassTest.scala @@ -25,21 +25,20 @@ package ai.privado.languageEngine.java.passes.config import ai.privado.cache.{AppCache, RuleCache} import ai.privado.entrypoint.PrivadoInput +import ai.privado.exporter.HttpConnectionMetadataExporter +import ai.privado.model.{Constants, Language} +import ai.privado.passes.PropertyParserPass import ai.privado.semantic.language.* -import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.testfixtures.JavaFrontendTestSuite import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, JavaProperty, Literal, Method, MethodParameterIn} +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import ai.privado.exporter.HttpConnectionMetadataExporter -import ai.privado.testfixtures.JavaFrontendTestSuite -import ai.privado.model.Constants class AnnotationTests extends JavaFrontendTestSuite { "ConfigFilePass" should { diff --git a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala index f0e438e41..91e9be94b 100644 --- a/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala +++ b/src/test/scala/ai/privado/languageEngine/java/passes/config/PropertyPassFilterTest.scala @@ -2,16 +2,16 @@ package ai.privado.languageEngine.java.passes.config import ai.privado.cache.{AppCache, RuleCache} import ai.privado.model.{ConfigAndRules, Language, SystemConfig} -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import io.shiftleft.semanticcpg.language.* -import ai.privado.semantic.language.* import scala.collection.mutable diff --git a/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala b/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala new file mode 100644 index 000000000..445f8995b --- /dev/null +++ b/src/test/scala/ai/privado/languageEngine/java/tagger/sink/DependencyTaggerTest.scala @@ -0,0 +1,459 @@ +package ai.privado.languageEngine.java.tagger.sink + +import ai.privado.cache.RuleCache +import ai.privado.exporter.SinkExporterValidator +import ai.privado.inputprocessor.DependencyInfo +import ai.privado.model +import ai.privado.model.* +import ai.privado.model.exporter.{DataFlowSubCategoryPathExcerptModel, SinkProcessingModel} +import ai.privado.testfixtures.JavaFrontendTestSuite + +import java.io.File +import scala.collection.mutable + +class DependencyTaggerTest extends JavaFrontendTestSuite with SinkExporterValidator { + + "Java pom.xml simple use case" should { + val dynamicRule = List( + RuleInfo( + "ThirdParties.SDK.twilio", + "twilio", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("www.twilio.com"), + List(".*(com.twilio.sdk).*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ), + RuleInfo( + "ThirdParties.SDK.Google.Sheets", + "Google Sheets", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("sheets.google.com"), + List(".*(com.google.apis).*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ) + ) + + val configAndRule = ConfigAndRules(sinks = dynamicRule) + val ruleCache = RuleCache().setRule(configAndRule) + val twilioDep = List( + DependencyInfo( + "com.twilio.sdk", + "twilio", + "8.19.1", + "twilio", + "ThirdParties.SDK.twilio", + "Twilio", + List("www.twilio.com"), + List(), + 32, + "pom.xml" + ), + DependencyInfo( + "com.google.apis", + "google-api-services-sheets", + "v4-rev493-1.23.0", + "google-api-services-sheets", + "ThirdParties.SDK.Google.Sheets", + "Google Sheets", + List("sheets.google.com"), + List(), + 42, + "pom.xml" + ) + ) + val cpg = code( + """ + | + | + | 4.0.0 + | + | org.springframework.boot + | spring-boot-starter-parent + | 2.5.4 + | + | + | com.atlan + | SMSService + | 0.0.1-SNAPSHOT + | SMSService + | Demo project for Spring Boot + | + | 1.8 + | 2020.0.3 + | + | + | + | mysql + | mysql-connector-java + | + | + | org.springframework.boot + | spring-boot-starter-web + | + | + | com.twilio.sdk + | twilio + | 8.19.1 + | + | + | org.springframework.boot + | spring-boot-starter-test + | test + | + | + | com.google.apis + | google-api-services-sheets + | v4-rev493-1.23.0 + | + | + | + | + | + | org.springframework.boot + | spring-boot-maven-plugin + | + | + | + | + |""".stripMargin, + "pom.xml" + ).withDependencies(twilioDep).withRuleCache(ruleCache) + + "3p dependency should get added in processing section" in { + val outjson = cpg.getPrivadoJson() + getSinks(outjson).size shouldBe 2 + val sinkProcessing = getSinkProcessings(outjson) + val sinkProceMap: mutable.Map[String, SinkProcessingModel] = mutable.Map[String, SinkProcessingModel]() + sinkProcessing.size shouldBe 2 + sinkProcessing.foreach(sink => { + sinkProceMap.put(sink.sinkId, sink) + }) + + val twilioSink = sinkProceMap.get("ThirdParties.SDK.twilio") + twilioSink shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.twilio", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = "twilio", + lineNumber = 32, + columnNumber = -1, + fileName = "pom.xml", + excerpt = + "\t\t\torg.springframework.boot\n\t\t\tspring-boot-starter-web\n\t\t\n\t\t\n\t\t\tcom.twilio.sdk\n\t\t\ttwilio /* <=== */ \n\t\t\t8.19.1\n\t\t\n\t\t\n\t\t\torg.springframework.boot\n\t\t\tspring-boot-starter-test", + arguments = None + ) + ) + ) + ) + + val gsheetSink = sinkProceMap.get("ThirdParties.SDK.Google.Sheets") + gsheetSink shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Google.Sheets", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = "google-api-services-sheets", + lineNumber = 42, + columnNumber = -1, + fileName = "pom.xml", + excerpt = + "\t\t\tspring-boot-starter-test\n\t\t\ttest\n\t\t\n \t\t\n\t\t\tcom.google.apis\n\t\t\tgoogle-api-services-sheets /* <=== */ \n\t\t\tv4-rev493-1.23.0\n\t\t\n\t\n\t\n\t\t", + arguments = None + ) + ) + ) + ) + } + } + + "Gradle tests" should { + val dynamicRule = List( + RuleInfo( + "ThirdParties.SDK.Google.Play.Services", + "Google Play Services", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("developers.google.com"), + List("(?i).*com.google.android.gms.*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ), + RuleInfo( + "ThirdParties.SDK.Firebase.Authentication", + "Firebase Authentication", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("firebase.google.com"), + List("(?i).*com.google.firebase.*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ), + RuleInfo( + "ThirdParties.SDK.Firebase", + "Firebase", + "Third Parties", + FilterProperty.METHOD_FULL_NAME, + Array("firebase.google.com"), + List("(?i).*com.google.firebase.*"), + false, + "", + Map(), + NodeType.REGULAR, + "", + CatLevelOne.SINKS, + "", + Language.JAVA, + Array() + ) + ) + + val configAndRule = ConfigAndRules(sinks = dynamicRule) + val ruleCache = RuleCache().setRule(configAndRule) + val twilioDep = List( + DependencyInfo( + "com.google.android.gms", + "play-services-auth", + "17.0.0", + " implementation 'com.google.android.gms:play-services-auth:17.0.0'\n", + "ThirdParties.SDK.Google.Play.Services", + "Google Play Services", + List("developers.google.com"), + List(), + 36, + "app/build.gradle" + ), + DependencyInfo( + "com.google.firebase", + "firebase-auth", + "19.0.0", + " implementation 'com.google.firebase:firebase-auth:19.0.0'\n", + "ThirdParties.SDK.Firebase.Authentication", + "Firebase Authentication", + List("firebase.google.com"), + List(), + 35, + "app/build.gradle" + ), + DependencyInfo( + "com.google.firebase", + "firebase-core", + "17.1.0", + " implementation 'com.google.firebase:firebase-core:17.1.0'\n", + "ThirdParties.SDK.Firebase", + "Firebase", + List("firebase.google.com"), + List(), + 32, + "app/build.gradle" + ), + DependencyInfo( + "com.google.gms", + "google-services", + "4.2.0", + " classpath 'com.google.gms:google-services:4.2.0'\n", + "ThirdParties.SDK.Google.Play.Services", + "Google Play Services", + List("play.google.com"), + List(), + 11, + "build.gradle" + ) + ) + val cpg = code( + """ + |buildscript { + | repositories { + | google() + | jcenter() + | + | } + | dependencies { + | classpath 'com.android.tools.build:gradle:3.5.3' + | classpath 'com.google.gms:google-services:4.2.0' + | + | // NOTE: Do not place your application dependencies here; they belong + | // in the individual module build.gradle files + | } + |} + | + |allprojects { + | repositories { + | google() + | jcenter() + | + | } + |} + | + |task clean(type: Delete) { + | delete rootProject.buildDir + |} + |""".stripMargin, + "build.gradle" + ).moreCode( + """ + |apply plugin: 'com.android.application' + |apply plugin: 'com.google.gms.google-services' + | + |android { + | compileSdkVersion 29 + | buildToolsVersion "29.0.1" + | defaultConfig { + | applicationId "com.example.edu" + | minSdkVersion 22 + | targetSdkVersion 29 + | versionCode 1 + | versionName "1.0" + | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + | } + | buildTypes { + | release { + | minifyEnabled false + | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + | } + | } + |} + | + |dependencies { + | implementation fileTree(dir: 'libs', include: ['*.jar']) + | implementation 'androidx.appcompat:appcompat:1.0.2' + | implementation 'androidx.constraintlayout:constraintlayout:1.1.3' + | implementation 'androidx.cardview:cardview:1.0.0' + | implementation 'com.android.support:appcompat-v7:29.0.0' + | implementation 'com.android.support:design:29.0.1' + | implementation 'androidx.appcompat:appcompat:1.0.0' + | implementation 'androidx.core:core:1.0.0' + | implementation 'com.google.firebase:firebase-core:17.1.0' + | implementation 'com.google.firebase:firebase-auth:19.0.0' + | implementation 'com.google.firebase:firebase-database:19.0.0' + | implementation 'com.google.firebase:firebase-auth:19.0.0' + | implementation 'com.google.android.gms:play-services-auth:17.0.0' + | implementation 'com.github.d-max:spots-dialog:1.1@aar' + | implementation 'com.google.android.material:material:1.0.0' + | implementation 'com.android.volley:volley:1.1.1' + | implementation 'com.google.firebase:firebase-messaging:20.1.0' + | implementation 'br.com.simplepass:loading-button-android:1.14.0' + | implementation 'com.google.android.material:material:1.0.0' + | implementation 'com.google.android.material:material:1.1.0-alpha09' + | implementation 'com.google.firebase:firebase-storage:16.0.4' + | implementation 'androidx.navigation:navigation-fragment:2.0.0' + | implementation 'androidx.navigation:navigation-ui:2.0.0' + | implementation 'androidx.lifecycle:lifecycle-extensions:2.0.0' + | + | testImplementation 'junit:junit:4.12' + | androidTestImplementation 'androidx.test:runner:1.2.0' + | androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + |} + |""".stripMargin, + List("app", "build.gradle").mkString(File.separator) + ).withDependencies(twilioDep) + .withRuleCache(ruleCache) + + "3p dependency should get added in processing section" in { + val outjson = cpg.getPrivadoJson() + getSinks(outjson).size shouldBe 3 + val sinkProcessing = getSinkProcessings(outjson) + val sinkProceMap: mutable.Map[String, SinkProcessingModel] = mutable.Map[String, SinkProcessingModel]() + sinkProcessing.size shouldBe 3 + sinkProcessing.foreach(sink => { + sinkProceMap.put(sink.sinkId, sink) + }) + + val playServices = sinkProceMap.get("ThirdParties.SDK.Google.Play.Services") + playServices shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Google.Play.Services", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = " classpath 'com.google.gms:google-services:4.2.0'\n", + lineNumber = 11, + columnNumber = -1, + fileName = "build.gradle", + excerpt = + " \n }\n dependencies {\n classpath 'com.android.tools.build:gradle:3.5.3'\n classpath 'com.google.gms:google-services:4.2.0'\n /* <=== */ \n // NOTE: Do not place your application dependencies here; they belong\n // in the individual module build.gradle files\n }\n}\n", + arguments = None + ), + DataFlowSubCategoryPathExcerptModel( + sample = " implementation 'com.google.android.gms:play-services-auth:17.0.0'\n", + lineNumber = 36, + columnNumber = -1, + fileName = "app/build.gradle", + excerpt = + " implementation 'androidx.appcompat:appcompat:1.0.0'\n implementation 'androidx.core:core:1.0.0'\n implementation 'com.google.firebase:firebase-core:17.1.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.firebase:firebase-database:19.0.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0' /* <=== */ \n implementation 'com.google.android.gms:play-services-auth:17.0.0'\n implementation 'com.github.d-max:spots-dialog:1.1@aar'\n implementation 'com.google.android.material:material:1.0.0'\n implementation 'com.android.volley:volley:1.1.1'\n implementation 'com.google.firebase:firebase-messaging:20.1.0'", + arguments = None + ) + ) + ) + ) + + val firebaseAuth = sinkProceMap.get("ThirdParties.SDK.Firebase.Authentication") + firebaseAuth shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Firebase.Authentication", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = " implementation 'com.google.firebase:firebase-auth:19.0.0'\n", + lineNumber = 35, + columnNumber = -1, + fileName = "app/build.gradle", + excerpt = + " implementation 'com.android.support:design:29.0.1'\n implementation 'androidx.appcompat:appcompat:1.0.0'\n implementation 'androidx.core:core:1.0.0'\n implementation 'com.google.firebase:firebase-core:17.1.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.firebase:firebase-database:19.0.0' /* <=== */ \n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.android.gms:play-services-auth:17.0.0'\n implementation 'com.github.d-max:spots-dialog:1.1@aar'\n implementation 'com.google.android.material:material:1.0.0'\n implementation 'com.android.volley:volley:1.1.1'", + arguments = None + ) + ) + ) + ) + + val firebase = sinkProceMap.get("ThirdParties.SDK.Firebase") + firebase shouldBe Some( + SinkProcessingModel( + sinkId = "ThirdParties.SDK.Firebase", + occurrences = List( + DataFlowSubCategoryPathExcerptModel( + sample = " implementation 'com.google.firebase:firebase-core:17.1.0'\n", + lineNumber = 32, + columnNumber = -1, + fileName = "app/build.gradle", + excerpt = + " implementation 'androidx.constraintlayout:constraintlayout:1.1.3'\n implementation 'androidx.cardview:cardview:1.0.0'\n implementation 'com.android.support:appcompat-v7:29.0.0'\n implementation 'com.android.support:design:29.0.1'\n implementation 'androidx.appcompat:appcompat:1.0.0'\n implementation 'androidx.core:core:1.0.0' /* <=== */ \n implementation 'com.google.firebase:firebase-core:17.1.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.firebase:firebase-database:19.0.0'\n implementation 'com.google.firebase:firebase-auth:19.0.0'\n implementation 'com.google.android.gms:play-services-auth:17.0.0'", + arguments = None + ) + ) + ) + ) + } + } +} diff --git a/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala index c69800b9d..c3d698cec 100644 --- a/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/javascript/passes/config/JSPropertiesFilePassTest.scala @@ -1,16 +1,16 @@ package ai.privado.languageEngine.javascript.passes.config import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions} -import io.joern.jssrc2cpg.{Config, JsSrc2Cpg} import io.joern.jssrc2cpg.passes.ImportsPass +import io.joern.jssrc2cpg.{Config, JsSrc2Cpg} import io.joern.x2cpg.X2Cpg import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.layers.LayerCreatorContext import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers diff --git a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala index d5dbe3d45..b7b828745 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyEnvPropertiesFilePassTest.scala @@ -1,12 +1,12 @@ package ai.privado.languageEngine.ruby.config import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* -import ai.privado.semantic.* import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.model.Language +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.* +import ai.privado.semantic.language.* import ai.privado.testfixtures.RubyFrontendTestSuite -import ai.privado.utility.PropertyParserPass import better.files.File import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala index 656b86768..41ee3fa07 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyPropertiesFilePassTestBase.scala @@ -3,7 +3,7 @@ package ai.privado.languageEngine.ruby.config import ai.privado.cache.RuleCache import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass import better.files.File import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala index 92d5468ed..314aeb389 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/config/RubyYamlPropertiesFilePassTest.scala @@ -1,11 +1,11 @@ package ai.privado.languageEngine.ruby.config import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* -import ai.privado.semantic.* import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.model.Language -import ai.privado.utility.PropertyParserPass +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.* +import ai.privado.semantic.language.* import better.files.File import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays diff --git a/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala index bf8772e49..662ffef89 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/monolith/MonolithTest.scala @@ -1,14 +1,6 @@ package ai.privado.languageEngine.ruby.monolith -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - RuleCache, - S3DatabaseDetailsCache, - TaggerCache -} +import ai.privado.cache.* import ai.privado.dataflow.Dataflow import ai.privado.entrypoint.PrivadoInput import ai.privado.exporter.monolith.MonolithExporter @@ -17,17 +9,17 @@ import ai.privado.languageEngine.go.tagger.source.IdentifierTagger import ai.privado.languageEngine.ruby.passes.config.RubyPropertyLinkerPass import ai.privado.languageEngine.ruby.tagger.monolith.MonolithTagger import ai.privado.model.{Constants, Language} +import ai.privado.passes.PropertyParserPass import ai.privado.rule.RuleInfoTestData -import ai.privado.utility.PropertyParserPass import better.files.File import io.joern.dataflowengineoss.language.Path import io.joern.rubysrc2cpg.{Config, RubySrc2Cpg} import io.joern.x2cpg.X2Cpg.applyDefaultOverlays import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import io.shiftleft.semanticcpg.language.* import scala.collection.mutable diff --git a/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala b/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala index 9bc9e87d5..7cb65d806 100644 --- a/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala +++ b/src/test/scala/ai/privado/languageEngine/ruby/tagger/schema/RubyMongoSchemaMapperTest.scala @@ -2,20 +2,12 @@ package ai.privado.languageEngine.ruby.tagger.schema import ai.privado.cache.{DatabaseDetailsCache, PropertyFilterCache, RuleCache} import ai.privado.languageEngine.ruby.RubyTestBase.* +import ai.privado.model.* +import ai.privado.passes.PropertyParserPass +import ai.privado.rule.RuleInfoTestData import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import ai.privado.model.{ - ConfigAndRules, - DatabaseColumn, - DatabaseDetails, - DatabaseSchema, - DatabaseTable, - Language, - SourceCodeModel -} -import ai.privado.rule.RuleInfoTestData -import ai.privado.utility.PropertyParserPass class RubyMongoSchemaMapperTest extends AnyWordSpec with Matchers with BeforeAndAfterAll { diff --git a/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala b/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala index bdbddd4f5..b0c189cad 100644 --- a/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala +++ b/src/test/scala/ai/privado/passes/SQLPropertyParserTest.scala @@ -1,19 +1,19 @@ package ai.privado.passes import ai.privado.cache.RuleCache -import ai.privado.semantic.language.* +import ai.privado.model.Language import ai.privado.model.sql.SQLQueryType +import ai.privado.passes.PropertyParserPass +import ai.privado.semantic.language.* import better.files.File +import io.joern.console.cpgcreation.guessLanguage import io.joern.jssrc2cpg.Config import io.joern.x2cpg.X2Cpg import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.semanticcpg.language.* import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import ai.privado.utility.PropertyParserPass -import ai.privado.model.Language -import io.joern.console.cpgcreation.guessLanguage -import io.shiftleft.semanticcpg.language._ class SQLPropertyParserTest extends AnyWordSpec with Matchers with BeforeAndAfterAll { "SQL Property parser" should { diff --git a/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala index 57c52c45f..4fe3e522c 100644 --- a/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.c.processor.CProcessor import ai.privado.model.Language @@ -24,7 +17,8 @@ class TestCpgWithC(val fileSuffix: String, val language: Language.Value) extends s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new CProcessor( ruleCache, diff --git a/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala index d57d67500..0132fee4c 100644 --- a/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.csharp.processor.CSharpProcessor import ai.privado.model.Language @@ -24,7 +17,8 @@ class TestCpgWithCSharp(val fileSuffix: String, val language: Language.Value) ex s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new CSharpProcessor( ruleCache, @@ -37,7 +31,8 @@ class TestCpgWithCSharp(val fileSuffix: String, val language: Language.Value) ex StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala index a94093968..9fe47ca50 100644 --- a/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.default.processor.DefaultProcessor import ai.privado.model.Language @@ -24,7 +17,8 @@ class TestCpgWithDefaultLanguage(val fileSuffix: String, val language: Language. s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new DefaultProcessor( ruleCache, diff --git a/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala index a17a33556..cf094b360 100644 --- a/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala @@ -1,18 +1,11 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.go.processor.GoProcessor -import ai.privado.model.{CatLevelOne, Constants, FilterProperty, Language, NodeType, RuleInfo} +import ai.privado.model.* import ai.privado.utility.StatsRecorder class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extends TestCpg { @@ -25,7 +18,8 @@ class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extend s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new GoProcessor( ruleCache, @@ -38,7 +32,8 @@ class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extend StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala index 7d698be5c..c0ca75a98 100644 --- a/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala @@ -2,10 +2,11 @@ package ai.privado.testfixtures import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.java.processor.JavaProcessor -import ai.privado.utility.StatsRecorder import ai.privado.model.Language +import ai.privado.utility.StatsRecorder class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) extends TestCpg { protected def getLanguageProcessor( @@ -16,7 +17,8 @@ class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) exte s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new JavaProcessor( ruleCache, @@ -29,7 +31,8 @@ class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) exte StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala index 9386aba21..ad41280d1 100644 --- a/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala @@ -2,9 +2,10 @@ package ai.privado.testfixtures import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.javascript.processor.JavascriptProcessor -import ai.privado.model.{CatLevelOne, Constants, FilterProperty, Language, NodeType, RuleInfo} +import ai.privado.model.* import ai.privado.utility.StatsRecorder class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value) extends TestCpg { @@ -16,7 +17,8 @@ class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new JavascriptProcessor( ruleCache, @@ -29,7 +31,8 @@ class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala index cbae891ff..77273f06f 100644 --- a/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.kotlin.processor.KotlinProcessor import ai.privado.model.Language @@ -24,7 +17,8 @@ class TestCpgWithKotlin(val fileSuffix: String, val language: Language.Value) ex s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new KotlinProcessor( ruleCache, @@ -37,7 +31,8 @@ class TestCpgWithKotlin(val fileSuffix: String, val language: Language.Value) ex StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala b/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala index 0d23cc45a..7cde290fc 100644 --- a/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala +++ b/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala @@ -2,6 +2,7 @@ package ai.privado.testfixtures import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.model.Language import ai.privado.rule.RuleInfoTestData @@ -21,6 +22,7 @@ trait LanguageFrontend { private var appCache: Option[AppCache] = None private var propertyFilterCache: Option[PropertyFilterCache] = None private var databaseDetailsCache: Option[DatabaseDetailsCache] = None + private var dependencies: Option[List[DependencyInfo]] = None def setPrivadoInput(privadoInput: PrivadoInput): Unit = { if (this.privadoInput.isDefined) { @@ -78,6 +80,13 @@ trait LanguageFrontend { this.databaseDetailsCache = Some(databaseDetailsCache) } + def setDependencies(dependencies: List[DependencyInfo]): Unit = { + if (this.dependencies.isDefined) { + throw new RuntimeException("Dependencies may only be set once per test") + } + this.dependencies = Some(dependencies) + } + protected def getProcessor(sourceCodePath: java.io.File): BaseProcessor = { val privadoInput = this.privadoInput.getOrElse(PrivadoInput()).copy(sourceLocation = Set(sourceCodePath.getAbsolutePath)) @@ -93,7 +102,8 @@ trait LanguageFrontend { this.s3DatabaseDetailsCache.getOrElse(S3DatabaseDetailsCache()), appCache, this.propertyFilterCache.getOrElse(PropertyFilterCache()), - this.databaseDetailsCache.getOrElse(DatabaseDetailsCache()) + this.databaseDetailsCache.getOrElse(DatabaseDetailsCache()), + this.dependencies.getOrElse(List()) ) } @@ -105,6 +115,7 @@ trait LanguageFrontend { s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor } diff --git a/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala index 1cbf91a58..adb4ac5dd 100644 --- a/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.php.processor.PhpProcessor import ai.privado.model.* @@ -24,7 +17,8 @@ class TestCpgWithPhp(val fileSuffix: String, val language: Language.Value) exten s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new PhpProcessor( ruleCache, @@ -37,7 +31,8 @@ class TestCpgWithPhp(val fileSuffix: String, val language: Language.Value) exten StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala index c347b3835..083d97b22 100644 --- a/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala @@ -1,20 +1,12 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.python.processor.PythonProcessor -import ai.privado.model.Language +import ai.privado.model.* import ai.privado.utility.StatsRecorder -import ai.privado.model.{CatLevelOne, Constants, FilterProperty, NodeType, RuleInfo} class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) extends TestCpg { @@ -26,7 +18,8 @@ class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) ex s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = new PythonProcessor( ruleCache, privadoInput, @@ -38,7 +31,8 @@ class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) ex StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala index 106a7a8e3..2c73e250e 100644 --- a/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala @@ -1,15 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{ - AppCache, - AuditCache, - DataFlowCache, - DatabaseDetailsCache, - PropertyFilterCache, - RuleCache, - S3DatabaseDetailsCache -} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor import ai.privado.languageEngine.ruby.processor.RubyProcessor import ai.privado.model.Language @@ -25,7 +18,8 @@ class TestCpgWithRuby(val fileSuffix: String, val language: Language.Value) exte s3DatabaseDetailsCache: S3DatabaseDetailsCache, appCache: AppCache, propertyFilterCache: PropertyFilterCache, - databaseDetailsCache: DatabaseDetailsCache + databaseDetailsCache: DatabaseDetailsCache, + dependencies: List[DependencyInfo] ): BaseProcessor = { new RubyProcessor( ruleCache, @@ -38,7 +32,8 @@ class TestCpgWithRuby(val fileSuffix: String, val language: Language.Value) exte StatsRecorder(), returnClosedCpg = false, databaseDetailsCache, - propertyFilterCache + propertyFilterCache, + dependencies = dependencies ) } } diff --git a/src/test/scala/ai/privado/testfixtures/TestCpg.scala b/src/test/scala/ai/privado/testfixtures/TestCpg.scala index d60100113..1afac108a 100644 --- a/src/test/scala/ai/privado/testfixtures/TestCpg.scala +++ b/src/test/scala/ai/privado/testfixtures/TestCpg.scala @@ -1,7 +1,8 @@ package ai.privado.testfixtures -import ai.privado.cache.{AppCache, AuditCache, DataFlowCache, PropertyFilterCache, RuleCache, S3DatabaseDetailsCache} +import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput +import ai.privado.inputprocessor.DependencyInfo import io.circe.Json import io.shiftleft.codepropertygraph.Cpg import overflowdb.Graph @@ -50,6 +51,11 @@ abstract class TestCpg extends Cpg() with TestCodeWriter with LanguageFrontend { this } + def withDependencies(dependencies: List[DependencyInfo]): this.type = { + setDependencies(dependencies) + this + } + def getPrivadoJson() = { graph _privadoJson.get From 2978b5252f1f5f24137a1ea29e64f8a4aa78bd8e Mon Sep 17 00:00:00 2001 From: KhemrajSingh Rathore Date: Fri, 23 Aug 2024 13:36:35 +0530 Subject: [PATCH 2/2] add test cases for js imports (#1258) * add test cases for js imports * fixing test cases * more test cases * skip running dataflow passes in metadata scan (#1259) * more test case * revert a change in postprocessing * skip applying post processing pass in metadata scan (#1260) * refactor test case * add tsconfig.json parsing logic * add cache * more log * logs * logs * more test case * introduce cache * introduce concurrent map * optimization * remove unused logs * use triemap * remove cache * refactoring * refactor * remove unnecessary test case * consider exclusion regex when reading files for tsconfig.json * fix - minor typo --- .../metadata/FileImportMappingPassJS.scala | 210 +++++++-- .../processor/JavascriptProcessor.scala | 2 +- .../scala/ai/privado/passes/JsonParser.scala | 48 ++ .../passes/JsonPropertyParserPass.scala | 45 +- .../FileImportMappingPassJSTests.scala | 425 ++++++++++++++++++ .../testfixtures/CFrontendTestSuite.scala | 6 +- .../CSharpFrontendTestSuite.scala | 6 +- .../DefaultFrontendTestSuite.scala | 6 +- .../testfixtures/GoFrontendTestSuite.scala | 6 +- .../testfixtures/JavaFrontendTestSuite.scala | 6 +- .../JavaScriptFrontendTestSuite.scala | 41 +- .../KotlinFrontendTestSuite.scala | 6 +- .../testfixtures/LanguageFrontend.scala | 12 + .../testfixtures/PhpFrontendTestSuite.scala | 6 +- .../PythonFrontendTestSuite.scala | 6 +- .../testfixtures/RubyFrontendTestSuite.scala | 6 +- .../ai/privado/testfixtures/TestCpg.scala | 10 + 17 files changed, 745 insertions(+), 102 deletions(-) create mode 100644 src/main/scala/ai/privado/passes/JsonParser.scala create mode 100644 src/test/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJSTests.scala diff --git a/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala b/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala index ea6684dbc..94ac4f601 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJS.scala @@ -1,19 +1,44 @@ package ai.privado.languageEngine.javascript.metadata -import ai.privado.cache.FileLinkingMetadata +import ai.privado.cache.{AppCache, FileLinkingMetadata, RuleCache} +import ai.privado.passes.JsonParser import io.joern.x2cpg.passes.frontend.XImportResolverPass import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Call import java.util.regex.{Matcher, Pattern} -import java.io.File as JFile -import scala.util.Try +import java.io.{FileNotFoundException, File as JFile} +import better.files.File +import better.files.File.VisitOptions +import io.joern.x2cpg.SourceFiles -class FileImportMappingPassJS(cpg: Cpg, fileLinkingMetadata: FileLinkingMetadata) - extends XImportResolverPass(cpg: Cpg) { +import java.util.concurrent.ConcurrentHashMap +import scala.collection.mutable +import scala.util.control.Breaks.{break, breakable} +import scala.util.{Failure, Success, Try} + +class FileImportMappingPassJS( + cpg: Cpg, + fileLinkingMetadata: FileLinkingMetadata, + appCache: AppCache, + ruleCache: RuleCache +) extends XImportResolverPass(cpg: Cpg) + with JsonParser { private val pathPattern = Pattern.compile("[\"']([\\w/.]+)[\"']") + val sep: String = Matcher.quoteReplacement(JFile.separator) + val root = s"${sanitiseProbeScanPath(codeRootDir)}${JFile.separator}" + + private val tsConfigPathMapping = mutable.HashMap[String, String]() + + private val tsConfigEntityMissCache = ConcurrentHashMap.newKeySet[String]() + + override def init(): Unit = { + // initialize tsconfig.json map + initializeConfigMap() + } + override protected def optionalResolveImport( fileName: String, importCall: Call, @@ -21,46 +46,155 @@ class FileImportMappingPassJS(cpg: Cpg, fileLinkingMetadata: FileLinkingMetadata importedAs: String, diffGraph: DiffGraphBuilder ): Unit = { - val pathSep = ":" - val rawEntity = importedEntity.stripPrefix("./") - val alias = importedAs - val matcher = pathPattern.matcher(rawEntity) - val sep = Matcher.quoteReplacement(JFile.separator) - val root = s"$codeRootDir${JFile.separator}" - val currentFile = s"$root$fileName" - val extension = better.files.File(currentFile).`extension`.getOrElse(".ts") + val pathSep = ":" + val currentFile = s"$root$fileName" + val extension = File(currentFile).`extension`.getOrElse(".ts") + val parentDirPath = File(currentFile).parent.pathAsString // Stores the path of the parent dir of current file + + val importedModule = getImportingModule(importedEntity, pathSep) + // We want to know if the import is local since if an external name is used to match internal methods we may have // false paths. - val isLocalImport = importedEntity.matches("^[.]+/?.*") - // TODO: At times there is an operation inside of a require, e.g. path.resolve(__dirname + "/../config/env/all.js") - // this tries to recover the string but does not perform string constant propagation - val entity = if (matcher.find()) matcher.group(1) else rawEntity - - val isImportingModule = !entity.contains(pathSep) - if (isLocalImport) { - val resolvedPath = Try( - better.files - .File(currentFile.stripSuffix(currentFile.split(sep).last), entity.split(pathSep).head) - .pathAsString - .stripPrefix(root) - ).getOrElse(entity) - fileLinkingMetadata.addToFileImportMap(fileName, s"$resolvedPath$extension") - } else { - val seperatedFilePathList = fileName.split(sep).toList - val startingModule = entity.split(sep).head - val moduleIndex = seperatedFilePathList.indexOf(startingModule) - if (moduleIndex != -1) { - Try { - val resolvedPath = better.files - .File(root, seperatedFilePathList.take(moduleIndex).mkString(sep), entity.split(pathSep).head) - .pathAsString - .stripPrefix(root) - fileLinkingMetadata.addToFileImportMap(fileName, s"$resolvedPath$extension") + val isRelativeImport = importedEntity.matches("^[.]+/?.*") + + if (isRelativeImport && importedModule.isDefined) { + getResolvedPath(parentDirPath, importedModule.get, importedAs, extension) match + case Failure(_) => // unable to resolve + case Success(resolvedPath) => fileLinkingMetadata.addToFileImportMap(fileName, resolvedPath) + } else if (importedModule.isDefined) { + val relativeDirCount = parentDirPath.stripPrefix(root).split(sep).length + breakable { + for (i <- 0 to relativeDirCount) { + val resolvedPath = + getResolvedPath( + parentDirPath.split(sep).dropRight(i).mkString(sep), + importedModule.get, + importedAs, + extension + ) + if (resolvedPath.isSuccess) { + fileLinkingMetadata.addToFileImportMap(fileName, resolvedPath.get) + break + } } } + } + } + + /** Function to get us a a probable relative file path, if exists for the importing module based on input parameters + * @param parentDirPath + * \- parent dir path where we intend to do the lookup + * @param relativePath + * \- importing module path + * @param importedAs + * \- importedAs value of import + * @param currentFileExtension + * \- current extension of the file being processed + * @return + */ + def getResolvedPath( + parentDirPath: String, + relativePath: String, + importedAs: String, + currentFileExtension: String + ): Try[String] = + Try { + val file = File(parentDirPath, relativePath) + if (file.exists && file.isRegularFile) { + file.pathAsString.stripPrefix(root) + } else { + // If not found, try to find a file with the same name extension + val baseName = file.nameWithoutExtension + val parentDir = file.parent + val fileWithSameNames = parentDir.list.filter { f => + f.isRegularFile && f.nameWithoutExtension == baseName + }.toList + + // If multiple files match with sameName, prefer the one having same extension + fileWithSameNames.size match + case size if size == 0 => throw FileNotFoundException() + case size if size == 1 => fileWithSameNames.head.pathAsString.stripPrefix(root) + case _ => + fileWithSameNames.find(f => f.`extension`.exists(_.equals(currentFileExtension))) match + case Some(fileWithSameNameAndExtension) => fileWithSameNameAndExtension.pathAsString.stripPrefix(root) + case None => fileWithSameNames.head.pathAsString.stripPrefix(root) + } + } + + /** From ImportedEntity after applying some lookup gives the importing Module + * @param importedEntity + * \- The value present as part of import or require statement + * @param pathSep + * \- The path sep used to concat the importing modules elements + * @return + */ + private def getImportingModule(importedEntity: String, pathSep: String) = { + importedEntity.split(pathSep).head match + case entity if entity.startsWith("@") => + // if import starts with `@` this can mean import of local modules in some case + if (tsConfigPathMapping.contains(entity)) { + Some(tsConfigPathMapping(entity)) + } else if (tsConfigEntityMissCache.contains(entity)) + None + else { + tsConfigPathMapping.keys.filter(_.endsWith("*")).find { k => + val keyRegex = k.replace("*", ".*").r + val value = keyRegex.matches(entity) + value + } match + case Some(configKey) => + val configPathValue = tsConfigPathMapping(configKey).stripSuffix("*") + val resolvedModule = entity.replace(configKey.stripSuffix("*"), configPathValue) + // println(s"ResolvedModule : $resolvedModule, for $entity and $importedEntity") + Some(resolvedModule) + case None => + tsConfigEntityMissCache.add(entity) + None + } + case entity => Some(entity) + } + + /** Returns all the file paths where tsconfig.json or jsconfig.json are defined + * @return + */ + private def getJsonPathConfigFiles: List[String] = { + val repoPath = sanitiseProbeScanPath(appCache.scanPath) + val filePaths = SourceFiles + .determine(repoPath, Set(".json"), ignoredFilesRegex = Option(ruleCache.getExclusionRegex.r))( + VisitOptions.default + ) + val filteredFilePaths = filePaths.filter { fp => + val f = File(fp) + f.nameWithoutExtension.contains("tsconfig") || f.nameWithoutExtension.contains("jsconfig") } + filteredFilePaths + } + /** Initializes the configuration map by reading the configurations files + */ + private def initializeConfigMap(): Unit = { + val compilerPathConstant = "compilerOptions.paths" + val configFilePaths = getJsonPathConfigFiles + + configFilePaths.foreach { configFilePath => + getJSONKeyValuePairs(configFilePath) + .filter(_._1.contains(compilerPathConstant)) + .foreach { pathEntry => + // do clean up of the paths key + // We would get keys like - compilerOptions.paths.@utils/*[0] + val pathKey = pathEntry._1.split(s"${compilerPathConstant}.").last.split("\\[").head + val pathValue = pathEntry._2 + tsConfigPathMapping.addOne(pathKey, pathValue) + } + } } + /** Function to sanitise the scanPath and remove `/probe` as we copy files in /probe folder, this helps in lookup for + * resolvedPaths + * @param scanPath + * \- repo path used for scanning + * @return + */ + private def sanitiseProbeScanPath(scanPath: String) = scanPath.replace(s"${sep}probe", "") } diff --git a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala index d8a584fb0..3aadb8158 100644 --- a/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala +++ b/src/main/scala/ai/privado/languageEngine/javascript/processor/JavascriptProcessor.scala @@ -119,7 +119,7 @@ class JavascriptProcessor( else JsSrc2Cpg.postProcessingPasses(cpg).foreach(_.createAndApply()) if (privadoInput.fileLinkingReport) { - new FileImportMappingPassJS(cpg, fileLinkingMetadata).createAndApply() + new FileImportMappingPassJS(cpg, fileLinkingMetadata, appCache, ruleCache).createAndApply() } } diff --git a/src/main/scala/ai/privado/passes/JsonParser.scala b/src/main/scala/ai/privado/passes/JsonParser.scala new file mode 100644 index 000000000..f90f2157f --- /dev/null +++ b/src/main/scala/ai/privado/passes/JsonParser.scala @@ -0,0 +1,48 @@ +package ai.privado.passes + +import io.circe.Json +import io.circe.parser.parse +import org.slf4j.LoggerFactory +import better.files.File + +trait JsonParser { + + /** Parses a JSON file and returns a list of key-value pairs for properties related to database connections and API + * endpoints. + * + * @param file + * the path to the JSON file to parse + * @return + * a list of key-value pairs where the keys match either the database connection or API endpoint naming conventions + */ + def getJSONKeyValuePairs(file: String): List[(String, String)] = { + val json = parse(File(file).contentAsString) + + // Recursively scan through the JSON to extract out all keys + def extractKeyValuePairs(json: Json, prefix: String = ""): List[(String, String)] = { + json match { + case obj if obj.isObject => + obj.asObject.get.toMap.toList.flatMap { case (key, value) => + val newPrefix = if (prefix.isEmpty) key else s"$prefix.$key" + extractKeyValuePairs(value, newPrefix) + } + case arr if arr.isArray => + arr.asArray.get.toList.zipWithIndex.flatMap { case (value, index) => + val newPrefix = s"$prefix[$index]" + extractKeyValuePairs(value, newPrefix) + } + case other => + List((prefix, other.asString.getOrElse(other.toString))) + } + } + + val keyValuePairs = json match { + case Right(jsonObject) => extractKeyValuePairs(jsonObject) + case Left(parsingError) => + List.empty + } + + keyValuePairs + } + +} diff --git a/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala b/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala index 3c6754881..f1faeccc7 100644 --- a/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala +++ b/src/main/scala/ai/privado/passes/JsonPropertyParserPass.scala @@ -15,9 +15,10 @@ import better.files.File import scala.collection.mutable import scala.io.Source import scala.util.{Try, Using} -class JsonPropertyParserPass(cpg: Cpg, projectRoot: String) extends PrivadoParallelCpgPass[String](cpg) { +class JsonPropertyParserPass(cpg: Cpg, projectRoot: String) + extends PrivadoParallelCpgPass[String](cpg) + with JsonParser { - val logger = LoggerFactory.getLogger(getClass) override def generateParts(): Array[String] = { val files = Try(File(projectRoot).listRecursively.filter(_.isRegularFile).map(_.path.toString).toArray).toOption @@ -40,44 +41,4 @@ class JsonPropertyParserPass(cpg: Cpg, projectRoot: String) extends PrivadoParal builder.addNode(propertyNode) propertyNode } - - /** Parses a JSON file and returns a list of key-value pairs for properties related to database connections and API - * endpoints. - * - * @param file - * the path to the JSON file to parse - * @return - * a list of key-value pairs where the keys match either the database connection or API endpoint naming conventions - */ - private def getJSONKeyValuePairs(file: String): List[(String, String)] = { - import better.files.File - val json = parse(File(file).contentAsString) - - // Recursively scan through the JSON to extract out all keys - def extractKeyValuePairs(json: Json, prefix: String = ""): List[(String, String)] = { - json match { - case obj if obj.isObject => - obj.asObject.get.toMap.toList.flatMap { case (key, value) => - val newPrefix = if (prefix.isEmpty) key else s"$prefix.$key" - extractKeyValuePairs(value, newPrefix) - } - case arr if arr.isArray => - arr.asArray.get.toList.zipWithIndex.flatMap { case (value, index) => - val newPrefix = s"$prefix[$index]" - extractKeyValuePairs(value, newPrefix) - } - case other => - List((prefix, other.asString.getOrElse(other.toString))) - } - } - - val keyValuePairs = json match { - case Right(jsonObject) => extractKeyValuePairs(jsonObject) - case Left(parsingError) => - logger.debug(parsingError.toString) - List.empty - } - - keyValuePairs - } } diff --git a/src/test/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJSTests.scala b/src/test/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJSTests.scala new file mode 100644 index 000000000..62585c4ea --- /dev/null +++ b/src/test/scala/ai/privado/languageEngine/javascript/metadata/FileImportMappingPassJSTests.scala @@ -0,0 +1,425 @@ +package ai.privado.languageEngine.javascript.metadata + +import ai.privado.cache.FileLinkingMetadata +import ai.privado.testfixtures.JavaScriptBaseCpgFrontendTestSuite + +import scala.collection.immutable.Set + +class FileImportMappingPassJSTests extends JavaScriptBaseCpgFrontendTestSuite { + + "File import mapping in javascript" should { + "resolve import files case 1" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from './module.js'; + |functionName(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from module'); + |} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 2" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import defaultExport from './module.js'; + |defaultExport(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export default function defaultExport() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 3" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import * as module from './module.js'; + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() {} + |export const someValue = 42; + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 4" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |const module = require('./module.js'); + |module(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports = function() { + | console.log('Function from module'); + |}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 5" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |const { functionName } = require('./module.js'); + |functionName(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports.functionName = function() {}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 6" ignore { + // TODO Failing as no import node created + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |require(['./module'], function(module) { + | module.functionName(); + |}); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports.functionName = function() {}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 7" ignore { + // TODO Failing as no import node created + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import('./module.js').then(module => { + | module.functionName(); + |}); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |module.exports.functionName = function() {}; + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 8" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from 'common/module'; + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 9" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from 'common/module'; + | + |""".stripMargin, + "src/util.js" + ).moreCode( + """ + |export function functionName() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 10" ignore { + // TODO Failing as no import node created + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |const functionName = () => import(/* webpackChunkName: "module" */ 'common/module'); + | + |""".stripMargin, + "src/util.js" + ).moreCode( + """ + |export function functionName() {} + | + |""".stripMargin, + "src/common/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/util.js") shouldBe Set("src/common/module.js") + } + + "resolve import files case 11" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from '../module.js'; + |functionName(); + | + |""".stripMargin, + "src/common/util.js" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from module'); + |} + | + |""".stripMargin, + "src/module.js" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.js") shouldBe Set("src/module.js") + } + } + + "File import mapping in TypeScript" should { + "resolve import files case 1" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from './module'; + |functionName(); + | + |""".stripMargin, + "src/common/util.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from module'); + |} + | + |""".stripMargin, + "src/common/module.ts" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 2" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import type { TypeName } from './module'; + | + |""".stripMargin, + "src/common/util.ts" + ).moreCode( + """ + |export type TypeName = { /* ... */ }; + | + |""".stripMargin, + "src/common/module.ts" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/common/util.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 3" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "common/module.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/module": ["common/module"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("common/module.ts") + } + + "resolve import files case 4" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName } from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "src/common/module.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/*": ["src/common/*"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 5" in { + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import * as module from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "src/common/module.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/*": ["src/common/*"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("src/common/module.ts") + } + + "resolve import files case 6" ignore { + // TODO Need to work on getting the export information from default CPG + val fileLinkingMetadata = FileLinkingMetadata() + val cpg = code( + """ + |import { functionName1 } from '@utils/module'; + |import { functionName2 } from '@utils/module'; + | + |""".stripMargin, + "src/main.ts" + ).moreCode( + """ + |export function functionName1() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "common/module/module1.ts" + ).moreCode( + """ + |export function functionName2() { + | console.log('Function from aliased path'); + |} + |""".stripMargin, + "common/module/module2.ts" + ).moreCode( + """ + |export * from './module1'; + |export * from './module2' + |""".stripMargin, + "common/module/index.ts" + ).moreCode( + """ + |{ + | "compilerOptions": { + | "baseUrl": "./", + | "paths": { + | "@utils/module": ["common/module"] + | } + | } + |} + |""".stripMargin, + "tsconfig.json" + ).withFileLinkingMetadata(fileLinkingMetadata) + + cpg.getFileLinkingData.getFileImportMap("src/main.ts") shouldBe Set("common/module/module1.ts") + } + } + +} diff --git a/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala index 4fe3e522c..021c38760 100644 --- a/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/CFrontendTestSuite.scala @@ -18,6 +18,7 @@ class TestCpgWithC(val fileSuffix: String, val language: Language.Value) extends appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new CProcessor( @@ -30,8 +31,9 @@ class TestCpgWithC(val fileSuffix: String, val language: Language.Value) extends appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata ) } } diff --git a/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala index 0132fee4c..c43df27c7 100644 --- a/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/CSharpFrontendTestSuite.scala @@ -18,6 +18,7 @@ class TestCpgWithCSharp(val fileSuffix: String, val language: Language.Value) ex appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new CSharpProcessor( @@ -30,8 +31,9 @@ class TestCpgWithCSharp(val fileSuffix: String, val language: Language.Value) ex appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala index 9fe47ca50..48fc965c9 100644 --- a/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/DefaultFrontendTestSuite.scala @@ -18,6 +18,7 @@ class TestCpgWithDefaultLanguage(val fileSuffix: String, val language: Language. appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new DefaultProcessor( @@ -30,8 +31,9 @@ class TestCpgWithDefaultLanguage(val fileSuffix: String, val language: Language. appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata ) } } diff --git a/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala index cf094b360..1b3027e7b 100644 --- a/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/GoFrontendTestSuite.scala @@ -19,6 +19,7 @@ class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extend appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new GoProcessor( @@ -31,8 +32,9 @@ class TestCpgWithGo(val fileSuffix: String, val language: Language.Value) extend appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala index c0ca75a98..0beec502d 100644 --- a/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/JavaFrontendTestSuite.scala @@ -18,6 +18,7 @@ class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) exte appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new JavaProcessor( @@ -30,8 +31,9 @@ class TestCpgWithJava(val fileSuffix: String, val language: Language.Value) exte appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala index ad41280d1..a4fe80d53 100644 --- a/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/JavaScriptFrontendTestSuite.scala @@ -4,7 +4,7 @@ import ai.privado.cache.* import ai.privado.entrypoint.PrivadoInput import ai.privado.inputprocessor.DependencyInfo import ai.privado.languageEngine.base.processor.BaseProcessor -import ai.privado.languageEngine.javascript.processor.JavascriptProcessor +import ai.privado.languageEngine.javascript.processor.{JavascriptProcessor, JavascriptBaseCPGProcessor} import ai.privado.model.* import ai.privado.utility.StatsRecorder @@ -18,6 +18,7 @@ class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new JavascriptProcessor( @@ -30,12 +31,46 @@ class TestCpgWithJavaScript(val fileSuffix: String, val language: Language.Value appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) } } +class TestCpgWithJavaScriptBase(val fileSuffix: String, val language: Language.Value) extends TestCpg { + protected def getLanguageProcessor( + ruleCache: RuleCache, + privadoInput: PrivadoInput, + dataFlowCache: DataFlowCache, + auditCache: AuditCache, + s3DatabaseDetailsCache: S3DatabaseDetailsCache, + appCache: AppCache, + propertyFilterCache: PropertyFilterCache, + databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, + dependencyInfo: List[DependencyInfo] + ): BaseProcessor = { + new JavascriptBaseCPGProcessor( + ruleCache, + privadoInput.copy(fileLinkingReport = true, isDeltaFileScan = true), + privadoInput.sourceLocation.head, + dataFlowCache, + auditCache, + s3DatabaseDetailsCache, + appCache, + StatsRecorder(), + returnClosedCpg = false, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata + ) + } +} + class JavaScriptFrontendTestSuite(fileSuffix: String = ".js", language: Language.Value = Language.JAVASCRIPT) extends PrivadoBaseTestFixture(() => new TestCpgWithJavaScript(fileSuffix, language)) {} + +class JavaScriptBaseCpgFrontendTestSuite(fileSuffix: String = ".js", language: Language.Value = Language.JAVASCRIPT) + extends PrivadoBaseTestFixture(() => new TestCpgWithJavaScriptBase(fileSuffix, language)) {} diff --git a/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala index 77273f06f..63ed2d330 100644 --- a/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/KotlinFrontendTestSuite.scala @@ -18,6 +18,7 @@ class TestCpgWithKotlin(val fileSuffix: String, val language: Language.Value) ex appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new KotlinProcessor( @@ -30,8 +31,9 @@ class TestCpgWithKotlin(val fileSuffix: String, val language: Language.Value) ex appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala b/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala index 7cde290fc..16d3651ee 100644 --- a/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala +++ b/src/test/scala/ai/privado/testfixtures/LanguageFrontend.scala @@ -22,6 +22,7 @@ trait LanguageFrontend { private var appCache: Option[AppCache] = None private var propertyFilterCache: Option[PropertyFilterCache] = None private var databaseDetailsCache: Option[DatabaseDetailsCache] = None + private var fileLinkingMetadata: Option[FileLinkingMetadata] = None private var dependencies: Option[List[DependencyInfo]] = None def setPrivadoInput(privadoInput: PrivadoInput): Unit = { @@ -80,6 +81,15 @@ trait LanguageFrontend { this.databaseDetailsCache = Some(databaseDetailsCache) } + def setFileLinkingMetadata(fileLinkingMetadata: FileLinkingMetadata): Unit = { + if (this.fileLinkingMetadata.isDefined) { + throw new RuntimeException("FileLinkingMetadata may only be set once per test") + } + this.fileLinkingMetadata = Some(fileLinkingMetadata) + } + + def getFileLinkingMetadata: FileLinkingMetadata = this.fileLinkingMetadata.getOrElse(FileLinkingMetadata()) + def setDependencies(dependencies: List[DependencyInfo]): Unit = { if (this.dependencies.isDefined) { throw new RuntimeException("Dependencies may only be set once per test") @@ -103,6 +113,7 @@ trait LanguageFrontend { appCache, this.propertyFilterCache.getOrElse(PropertyFilterCache()), this.databaseDetailsCache.getOrElse(DatabaseDetailsCache()), + this.fileLinkingMetadata.getOrElse(FileLinkingMetadata()), this.dependencies.getOrElse(List()) ) } @@ -116,6 +127,7 @@ trait LanguageFrontend { appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor } diff --git a/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala index adb4ac5dd..2000f0037 100644 --- a/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/PhpFrontendTestSuite.scala @@ -18,6 +18,7 @@ class TestCpgWithPhp(val fileSuffix: String, val language: Language.Value) exten appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new PhpProcessor( @@ -30,8 +31,9 @@ class TestCpgWithPhp(val fileSuffix: String, val language: Language.Value) exten appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala index 083d97b22..d8e150f6a 100644 --- a/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/PythonFrontendTestSuite.scala @@ -19,6 +19,7 @@ class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) ex appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = new PythonProcessor( ruleCache, @@ -30,8 +31,9 @@ class TestCpgWithPython(val fileSuffix: String, val language: Language.Value) ex appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) diff --git a/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala b/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala index 2c73e250e..47ffb79ca 100644 --- a/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala +++ b/src/test/scala/ai/privado/testfixtures/RubyFrontendTestSuite.scala @@ -19,6 +19,7 @@ class TestCpgWithRuby(val fileSuffix: String, val language: Language.Value) exte appCache: AppCache, propertyFilterCache: PropertyFilterCache, databaseDetailsCache: DatabaseDetailsCache, + fileLinkingMetadata: FileLinkingMetadata, dependencies: List[DependencyInfo] ): BaseProcessor = { new RubyProcessor( @@ -31,8 +32,9 @@ class TestCpgWithRuby(val fileSuffix: String, val language: Language.Value) exte appCache, StatsRecorder(), returnClosedCpg = false, - databaseDetailsCache, - propertyFilterCache, + databaseDetailsCache = databaseDetailsCache, + propertyFilterCache = propertyFilterCache, + fileLinkingMetadata = fileLinkingMetadata, dependencies = dependencies ) } diff --git a/src/test/scala/ai/privado/testfixtures/TestCpg.scala b/src/test/scala/ai/privado/testfixtures/TestCpg.scala index 1afac108a..b3335adc5 100644 --- a/src/test/scala/ai/privado/testfixtures/TestCpg.scala +++ b/src/test/scala/ai/privado/testfixtures/TestCpg.scala @@ -51,6 +51,11 @@ abstract class TestCpg extends Cpg() with TestCodeWriter with LanguageFrontend { this } + def withFileLinkingMetadata(fileLinkingMetadata: FileLinkingMetadata): this.type = { + setFileLinkingMetadata(fileLinkingMetadata) + this + } + def withDependencies(dependencies: List[DependencyInfo]): this.type = { setDependencies(dependencies) this @@ -61,6 +66,11 @@ abstract class TestCpg extends Cpg() with TestCodeWriter with LanguageFrontend { _privadoJson.get } + def getFileLinkingData: FileLinkingMetadata = { + graph + this.getFileLinkingMetadata + } + override def graph: Graph = { if (_graph.isEmpty) { val codeDir = writeCode(fileSuffix)