Merge branch 'dev' into property-pass-improvement

.github/workflows/code-quality.yml

@@ -35,6 +35,6 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: Run unit test
-        run: sbt test test:test
+        run: sbt -J-Xmx4G test test:test
       - run: echo "Previous step failed because unit test failed."
-        if: ${{ failure() }}
+        if: ${{ failure() }}

Dockerfile

@@ -1,6 +1,6 @@
 ARG VERSION=1.0.0
 FROM openjdk:18.0.2.1-jdk-bullseye as build
-RUN apt update && apt install -y python3 git curl bash ruby-full
+RUN apt update && apt install -y python3 git curl bash ruby-full php
 RUN ln -sf python3 /usr/bin/python
 ENV SBT_VERSION 1.7.1
 ENV SBT_HOME /usr/local/sbt

README.md

@@ -1,7 +1,7 @@
 Privado Core
 =============================================
 
-Branch structure
+Branch structure 
 
 main - This branch will contain the released version of the code.
 

build.sbt

@@ -1,4 +1,5 @@
 import sbt.Credentials
+import better.files.File
 
 name                     := "privado-core"
 ThisBuild / organization := "ai.privado"
@@ -206,12 +207,42 @@ stage := Def
 
 Compile / compile := ((Compile / compile) dependsOn dotnetAstGenDlTask).value
 
-// Also remove astgen binaries with clean, e.g., to allow for updating them.
+// Download php-parser: start
+// This is based on how php2cpg vendors the php-parser in Joern
+val phpParserVersion       = "4.15.7"
+val upstreamParserBinName  = "php-parser.phar"
+val versionedParserBinName = s"php-parser-$phpParserVersion.phar"
+val phpParserDlUrl =
+  s"https://github.com/joernio/PHP-Parser/releases/download/v$phpParserVersion/$upstreamParserBinName"
+
+Compile / compile := ((Compile / compile) dependsOn phpParseDlTask).value
+
+lazy val phpParseDlTask = taskKey[Unit]("Download php-parser binaries")
+phpParseDlTask := {
+  val phpBinDir = baseDirectory.value / "bin" / "php-parser"
+  phpBinDir.mkdirs()
+
+  val downloadedFile = SimpleCache.downloadMaybe(phpParserDlUrl)
+  IO.copyFile(downloadedFile, phpBinDir / versionedParserBinName)
+
+  File((phpBinDir / "php-parser.php").getPath)
+    .createFileIfNotExists()
+    .overwrite(s"<?php\nrequire('$versionedParserBinName');?>")
+
+  val distDir = (Universal / stagingDirectory).value / "bin" / "php-parser"
+  distDir.mkdirs()
+  IO.copyDirectory(phpBinDir, distDir)
+}
+// Download php-parser: end
+
+// Also remove astgen and php-parser binaries with clean, e.g., to allow for updating them.
 // Sadly, we can't define the bin/ folders globally,
 // as .value can only be used within a task or setting macro
 cleanFiles ++= Seq(
   baseDirectory.value / "bin" / "astgen",
-  (Universal / stagingDirectory).value / "bin" / "astgen"
+  (Universal / stagingDirectory).value / "bin" / "astgen",
+  baseDirectory.value / "bin" / "php-parser",
+  (Universal / stagingDirectory).value / "bin" / "php-parser"
 ) ++ astGenBinaryNames.map(fileName => SimpleCache.encodeFile(s"$astGenDlUrl$fileName"))
 Compile / doc / sources                := Seq.empty
 Compile / packageDoc / publishArtifact := false

src/main/scala/ai/privado/entrypoint/CommandParser.scala

@@ -43,6 +43,7 @@ case class PrivadoInput(
   disableReadDataflow: Boolean = false,
   enableAPIDisplay: Boolean = false,
   enableLambdaFlows: Boolean = false,
+  enableAPIByParameter: Boolean = false,
   ignoreExcludeRules: Boolean = false,
   ignoreSinkSkipRules: Boolean = false,
   skipUpload: Boolean = false,
@@ -85,6 +86,8 @@ object CommandConstants {
   val ENABLE_API_DISPLAY_ABBR                      = "ead"
   val ENABLE_LAMBDA_FLOWS                          = "enable-lambda-flows"
   val ENABLE_LAMBDA_FLOWS_ABBR                     = "elf"
+  val ENABLE_API_BY_PARAMETER                      = "enable-api-by-parameter"
+  val ENABLE_API_BY_PARAMETER_ABBR                 = "eabyp"
   val IGNORE_EXCLUDE_RULES                         = "ignore-exclude-rules"
   val IGNORE_EXCLUDE_RULES_ABBR                    = "ier"
   val UPLOAD                                       = "upload"
@@ -211,6 +214,11 @@ object CommandParser {
               .optional()
               .action((_, c) => c.copy(enableLambdaFlows = true))
               .text("Enable lambda flows"),
+            opt[Unit](CommandConstants.ENABLE_API_BY_PARAMETER)
+              .abbr(CommandConstants.ENABLE_API_BY_PARAMETER_ABBR)
+              .optional()
+              .action((_, c) => c.copy(enableAPIByParameter = true))
+              .text("Enable API tagging by parameter name match"),
             opt[Unit](CommandConstants.IGNORE_EXCLUDE_RULES)
               .abbr(CommandConstants.IGNORE_EXCLUDE_RULES_ABBR)
               .optional()

src/main/scala/ai/privado/entrypoint/ScanProcessor.scala

@@ -31,6 +31,7 @@ import ai.privado.languageEngine.ruby.processor.RubyProcessor
 import ai.privado.languageEngine.default.processor.DefaultProcessor
 import ai.privado.languageEngine.kotlin.processor.KotlinProcessor
 import ai.privado.languageEngine.go.processor.GoProcessor
+import ai.privado.languageEngine.php.processor.PhpProcessor
 import ai.privado.metric.MetricHandler
 import ai.privado.model.Language.Language
 import ai.privado.model.*
@@ -430,6 +431,18 @@ object ScanProcessor extends CommandProcessor {
                   auditCache,
                   s3DatabaseDetailsCache
                 ).processCpg()
+              case language if language == Languages.PHP =>
+                println(s"${Calendar.getInstance().getTime} - Detected language 'PHP'")
+                new PhpProcessor(
+                  getProcessedRule(Set(Language.PHP)),
+                  this.config,
+                  sourceRepoLocation,
+                  Language.PHP,
+                  dataFlowCache = getDataflowCache,
+                  auditCache,
+                  s3DatabaseDetailsCache
+                )
+                  .processCpg()
               case _ =>
                 if (checkJavaSourceCodePresent(sourceRepoLocation)) {
                   println(

src/main/scala/ai/privado/exporter/HttpConnectionMetadataExporter.scala

@@ -38,14 +38,20 @@ class HttpConnectionMetadataExporter(cpg: Cpg, ruleCache: RuleCache) {
   private val FEIGN_CLIENT              = "FeignClient"
   private val SPRING_ANNOTATION_ID      = "Collections.Annotation.Spring"
   private val STRING_START_WITH_SLASH   = "/.{2,}"
+  private val STRING_CONTAINS_URL       = ".*\\.[a-z]{2,5}/[a-z]{2,}.*"
   private val STRING_CONTAINS_TWO_SLASH = ".*/.*/.*"
   private val SPRING_APPLICATION_BASE_PATH =
     "(?i)(server[.]servlet[.]context-path|server[.]servlet[.]contextPath)|(spring[.]application[.]name)"
   private val URL_PATH_WITH_VARIABLE_SYMBOLS                       = "^(?=.*/)[${}/\"'a-zA-Z0-9:.,%?_=]+"
   private val ALPHABET                                             = "[a-zA-Z]"
   private val STRING_WITH_CONSECUTIVE_DOTS_OR_DOT_SLASH_OR_NEWLINE = "(?s).*(\\.\\.|\\./|\n).*"
   private val ESCAPE_STRING_SLASHES                                = "(\\\")"
-  private val IMPORT_REGEX_WITH_SLASHES                            = "(?s)^(?=.*/)(?!.*/$).*"
+  //  Regex to eliminate pattern ending with file suffix
+  //  Demo: https://regex101.com/r/ojV93D/1
+  private val FILE_SUFFIX_REGEX_PATTERN = ".*[.][a-z]{2,5}(\\\")?$"
+  private val SUFFIX_PATTERN            = "^(\\.\\/|\\.\\.|\\/\\/).*"
+  private val COMMON_FALSE_POSITIVE_EGRESS_PATTERN =
+    ".*(BEGIN PRIVATE KEY|sha512|googleapis|sha1|amazonaws|github|</div>|</p>|<img|<class|require\\(|\\s).*"
 
   private val SLASH_SYMBOL          = "/"
   private val FORMAT_STRING_SYMBOLS = "[{}]"
@@ -101,7 +107,14 @@ class HttpConnectionMetadataExporter(cpg: Cpg, ruleCache: RuleCache) {
     var egressUrls = List[String]()
 
     egressUrls = egressUrls.concat(
-      cpg.property.or(_.value(STRING_START_WITH_SLASH), _.value(STRING_CONTAINS_TWO_SLASH)).value.dedup.l
+      cpg.property
+        .filterNot(_.value.matches(FILE_SUFFIX_REGEX_PATTERN))
+        .filterNot(_.value.matches(COMMON_FALSE_POSITIVE_EGRESS_PATTERN))
+        .filterNot(_.value.matches(SUFFIX_PATTERN))
+        .or(_.value(STRING_START_WITH_SLASH), _.value(STRING_CONTAINS_TWO_SLASH), _.value(STRING_CONTAINS_URL))
+        .value
+        .dedup
+        .l
     )
 
     egressUrls = egressUrls.concat(addUrlFromFeignClient())
@@ -128,9 +141,11 @@ class HttpConnectionMetadataExporter(cpg: Cpg, ruleCache: RuleCache) {
       val ruleInfo = ruleCache.getRule.collections
         .filter(_.catLevelTwo == Constants.annotations)
         .filter(_.id == SPRING_ANNOTATION_ID)
-        .head
+        .headOption
+
+      if (ruleInfo.isEmpty) return egressUrls
 
-      val combinedRulePatterns = ruleInfo.combinedRulePattern
+      val combinedRulePatterns = ruleInfo.get.combinedRulePattern
 
 //       filters these annotation to include only those found in files that contain a FeignClient,
 //       producing a list of matched annotations

src/main/scala/ai/privado/exporter/RepoConfigMetaDataExporter.scala

@@ -8,19 +8,28 @@ import io.shiftleft.semanticcpg.language.*
 import ai.privado.languageEngine.java.language.NodeStarters
 import ai.privado.model.Constants
 
+import scala.collection.immutable.HashMap
+
 object RepoConfigMetaDataExporter {
 
   def getMetaData(cpg: Cpg, ruleCache: RuleCache) = {
-    val metaData   = mutable.LinkedHashMap[String, String]()
     val configRule = ruleCache.getSystemConfigByKey(Constants.RepoPropertyConfig)
     try {
-      val propertySources = cpg.property.filter(p => p.name matches configRule).l
-      propertySources.foreach(p => {
-        metaData.put(p.name, p.value)
-      })
+      cpg.property
+        .filter(p => p.name matches configRule)
+        .l
+        .map(p => {
+          HashMap(
+            Constants.name     -> p.name,
+            Constants.value    -> p.value,
+            Constants.filePath -> p.file.name.headOption.getOrElse("")
+          )
+        })
     } catch {
-      case ex: Exception => println("Error while fetching repo config metadata")
+      case ex: Exception => {
+        println("Error while fetching repo config metadata")
+        List()
+      }
     }
-    metaData
   }
 }

src/main/scala/ai/privado/languageEngine/java/passes/config/JavaAnnotationPropertyLinkerPass.scala

@@ -10,58 +10,82 @@ class JavaAnnotationPropertyLinkerPass(cpg: Cpg) extends PrivadoParallelCpgPass[
 
   override def generateParts(): Array[_ <: AnyRef] = {
     cpg.annotation
-      .nameExact("Value")
+      .name(".*(Value|Named).*")
       .filter(_.parameterAssign.nonEmpty)
       .toArray
   }
 
   override def runOnPart(builder: DiffGraphBuilder, annotation: Annotation): Unit = {
 
-    /** List of all parameters annotated with Spring's `Value` annotation, along with the property name.
-      */
-    if (annotation.parameterAssign.code("\\\"\\$\\{.*\\}\\\"").nonEmpty && annotation.parameter.nonEmpty) {
-      val literalName = annotation.parameterAssign.code.head
-      val value       = Option(literalName.slice(3, literalName.length - 2)).getOrElse("")
+    if (annotation.name == "Value") {
+
+      /** List of all parameters annotated with Spring's `Value` annotation, along with the property name.
+        */
+      if (annotation.parameterAssign.code("\\\"\\$\\{.*\\}\\\"").nonEmpty && annotation.parameter.nonEmpty) {
+        val literalName = annotation.parameterAssign.code.head
+        val value       = Option(literalName.slice(3, literalName.length - 2)).getOrElse("")
+        if (value.nonEmpty) {
+          cpg.property
+            .filter(p => p.name == value)
+            .foreach(p => {
+              connectEnvProperty(annotation.parameter.head, p, builder)
+            })
+        }
+      }
+
+      /** List of all parameters annotated with Spring's `Value` annotation, along with the property name.
+        */
+      if (annotation.member.nonEmpty) {
+        cpg.property
+          .filter(p =>
+            p.name == Option(
+              annotation.parameterAssign.head.code.slice(3, annotation.parameterAssign.head.code.length - 2)
+            ).getOrElse("")
+          )
+          .foreach(p => {
+            connectEnvProperty(annotation.member.head, p, builder)
+          })
+      }
+
+      /** List of all methods annotated with Spring's `Value` annotation, along with the method node
+        */
+      if (annotation.method.nonEmpty) {
+        val key = annotation.parameterAssign.head
+        cpg.property
+          .filter(p => p.name == Option(key.code.slice(3, key.code.length - 2)).getOrElse(""))
+          .foreach(p => {
+            val referenceMember = annotation.method.head.ast.fieldAccess.referencedMember.l.headOption.orNull
+            if (referenceMember != null) {
+              connectEnvProperty(referenceMember, p, builder)
+            }
+          })
+      }
+    } else if (annotation.name == "Named" && annotation.parameter.nonEmpty) {
+      val value = annotation.parameterAssign.code.head.split("[.]").lastOption.getOrElse("")
       if (value.nonEmpty) {
         cpg.property
-          .filter(p => p.name == value)
+          .filter(p => p.name.endsWith(value))
           .foreach(p => {
             connectEnvProperty(annotation.parameter.head, p, builder)
           })
       }
     }
-
-    /** List of all parameters annotated with Spring's `Value` annotation, along with the property name.
-      */
-    if (annotation.member.nonEmpty) {
-      cpg.property
-        .filter(p =>
-          p.name == Option(
-            annotation.parameterAssign.head.code.slice(3, annotation.parameterAssign.head.code.length - 2)
-          ).getOrElse("")
-        )
-        .foreach(p => {
-          connectEnvProperty(annotation.member.head, p, builder)
-        })
-    }
-
-    /** List of all methods annotated with Spring's `Value` annotation, along with the method node
-      */
-    if (annotation.method.nonEmpty) {
-      val key = annotation.parameterAssign.head
-      cpg.property
-        .filter(p => p.name == Option(key.code.slice(3, key.code.length - 2)).getOrElse(""))
-        .foreach(p => {
-          val referenceMember = annotation.method.head.ast.fieldAccess.referencedMember.l.headOption.orNull
-          if (referenceMember != null) {
-            connectEnvProperty(referenceMember, p, builder)
-          }
-        })
-    }
   }
 
   def connectEnvProperty(literalNode: AstNode, propertyNode: JavaProperty, builder: DiffGraphBuilder): Unit = {
     builder.addEdge(propertyNode, literalNode, EdgeTypes.IS_USED_AT)
     builder.addEdge(literalNode, propertyNode, EdgeTypes.ORIGINAL_PROPERTY)
   }
 }
+
+//private def namedAnnotatedParameters(): List[(MethodParameterIn, String)] = cpg.annotation
+//  .nameExact("Named")
+//  .filter(_.parameter.nonEmpty)
+//  .map { x =>
+//    val value = x.parameterAssign.code.next().split("[.]").lastOption.getOrElse("")
+//    (x.parameter.next(), value)
+//  }
+//  .filter { (_, value) =>
+//    value.nonEmpty
+//  }
+//  .toList

src/main/scala/ai/privado/languageEngine/java/passes/config/JavaPropertyLinkerPass.scala

@@ -0,0 +1 @@
+

src/main/scala/ai/privado/languageEngine/java/tagger/PrivadoTagger.scala

@@ -40,6 +40,7 @@ import ai.privado.languageEngine.java.tagger.collection.{
   SOAPCollectionTagger
 }
 import ai.privado.languageEngine.java.tagger.config.JavaDBConfigTagger
+import ai.privado.languageEngine.java.tagger.sink.api.JavaAPISinkTagger
 import ai.privado.languageEngine.java.tagger.sink.{InheritMethodTagger, JavaAPITagger, MessagingConsumerCustomTagger}
 import ai.privado.languageEngine.java.tagger.source.{IdentifierTagger, InSensitiveCallTagger}
 import ai.privado.tagger.PrivadoBaseTagger
@@ -82,6 +83,8 @@ class PrivadoTagger(cpg: Cpg) extends PrivadoBaseTagger {
 
     new JavaS3Tagger(cpg, s3DatabaseDetailsCache).createAndApply()
 
+    JavaAPISinkTagger.applyTagger(cpg, ruleCache, privadoInputConfig)
+
     new JavaAPITagger(cpg, ruleCache, privadoInputConfig).createAndApply()
     // Custom Rule tagging
     if (!privadoInputConfig.ignoreInternalRules) {

src/main/scala/ai/privado/languageEngine/java/tagger/sink/FeignAPI.scala

@@ -201,7 +201,9 @@ class FeignAPI(cpg: Cpg, ruleCache: RuleCache) {
           apiCalls.foreach(apiNode => {
             val domain         = getDomainFromString(apiLiteral)
             val newRuleIdToUse = ruleInfo.id + "." + domain
-            ruleCache.setRuleInfo(ruleInfo.copy(id = newRuleIdToUse, name = ruleInfo.name + " " + domain))
+            ruleCache.setRuleInfo(
+              ruleInfo.copy(id = newRuleIdToUse, name = ruleInfo.name + " " + domain, isGenerated = true)
+            )
             addRuleTags(builder, apiNode, ruleInfo, ruleCache, Some(newRuleIdToUse))
             storeForTag(builder, apiNode, ruleCache)(Constants.apiUrl + newRuleIdToUse, apiLiteral)
           })