Release PR

src/main/scala/ai/privado/exporter/HttpConnectionMetadataExporter.scala

@@ -38,14 +38,20 @@ class HttpConnectionMetadataExporter(cpg: Cpg, ruleCache: RuleCache) {
   private val FEIGN_CLIENT              = "FeignClient"
   private val SPRING_ANNOTATION_ID      = "Collections.Annotation.Spring"
   private val STRING_START_WITH_SLASH   = "/.{2,}"
+  private val STRING_CONTAINS_URL       = ".*\\.[a-z]{2,5}/[a-z]{2,}.*"
   private val STRING_CONTAINS_TWO_SLASH = ".*/.*/.*"
   private val SPRING_APPLICATION_BASE_PATH =
     "(?i)(server[.]servlet[.]context-path|server[.]servlet[.]contextPath)|(spring[.]application[.]name)"
   private val URL_PATH_WITH_VARIABLE_SYMBOLS                       = "^(?=.*/)[${}/\"'a-zA-Z0-9:.,%?_=]+"
   private val ALPHABET                                             = "[a-zA-Z]"
   private val STRING_WITH_CONSECUTIVE_DOTS_OR_DOT_SLASH_OR_NEWLINE = "(?s).*(\\.\\.|\\./|\n).*"
   private val ESCAPE_STRING_SLASHES                                = "(\\\")"
-  private val IMPORT_REGEX_WITH_SLASHES                            = "(?s)^(?=.*/)(?!.*/$).*"
+  //  Regex to eliminate pattern ending with file suffix
+  //  Demo: https://regex101.com/r/ojV93D/1
+  private val FILE_SUFFIX_REGEX_PATTERN = ".*[.][a-z]{2,5}(\\\")?$"
+  private val SUFFIX_PATTERN            = "^(\\.\\/|\\.\\.|\\/\\/).*"
+  private val COMMON_FALSE_POSITIVE_EGRESS_PATTERN =
+    ".*(BEGIN PRIVATE KEY|sha512|googleapis|sha1|amazonaws|github|</div>|</p>|<img|<class|require\\(|\\s).*"
 
   private val SLASH_SYMBOL          = "/"
   private val FORMAT_STRING_SYMBOLS = "[{}]"
@@ -101,7 +107,14 @@ class HttpConnectionMetadataExporter(cpg: Cpg, ruleCache: RuleCache) {
     var egressUrls = List[String]()
 
     egressUrls = egressUrls.concat(
-      cpg.property.or(_.value(STRING_START_WITH_SLASH), _.value(STRING_CONTAINS_TWO_SLASH)).value.dedup.l
+      cpg.property
+        .filterNot(_.value.matches(FILE_SUFFIX_REGEX_PATTERN))
+        .filterNot(_.value.matches(COMMON_FALSE_POSITIVE_EGRESS_PATTERN))
+        .filterNot(_.value.matches(SUFFIX_PATTERN))
+        .or(_.value(STRING_START_WITH_SLASH), _.value(STRING_CONTAINS_TWO_SLASH), _.value(STRING_CONTAINS_URL))
+        .value
+        .dedup
+        .l
     )
 
     egressUrls = egressUrls.concat(addUrlFromFeignClient())
@@ -128,9 +141,11 @@ class HttpConnectionMetadataExporter(cpg: Cpg, ruleCache: RuleCache) {
       val ruleInfo = ruleCache.getRule.collections
         .filter(_.catLevelTwo == Constants.annotations)
         .filter(_.id == SPRING_ANNOTATION_ID)
-        .head
+        .headOption
+
+      if (ruleInfo.isEmpty) return egressUrls
 
-      val combinedRulePatterns = ruleInfo.combinedRulePattern
+      val combinedRulePatterns = ruleInfo.get.combinedRulePattern
 
 //       filters these annotation to include only those found in files that contain a FeignClient,
 //       producing a list of matched annotations

src/main/scala/ai/privado/languageEngine/java/passes/config/JavaPropertyLinkerPass.scala

@@ -52,10 +52,10 @@ class JavaPropertyLinkerPass(cpg: Cpg) extends PrivadoParallelCpgPass[JavaProper
   }
 
   private def connectAnnotatedParameters(propertyNode: JavaProperty, builder: BatchedUpdate.DiffGraphBuilder): Unit = {
-    val paramsAndValues = annotatedParameters()
+    val paramsAndValues = annotatedParameters() ++ namedAnnotatedParameters()
 
     paramsAndValues.iterator
-      .filter { case (_, value) => propertyNode.name == value }
+      .filter { case (_, value) => propertyNode.name == value || propertyNode.name.endsWith(s".$value") }
       .foreach { case (param, _) =>
         builder.addEdge(propertyNode, param, EdgeTypes.IS_USED_AT)
         builder.addEdge(param, propertyNode, EdgeTypes.ORIGINAL_PROPERTY)
@@ -96,6 +96,18 @@ class JavaPropertyLinkerPass(cpg: Cpg) extends PrivadoParallelCpgPass[JavaProper
     .map { x => (x.parameterAssign.next(), x.method.next()) }
     .toList
 
+  private def namedAnnotatedParameters(): List[(MethodParameterIn, String)] = cpg.annotation
+    .nameExact("Named")
+    .filter(_.parameter.nonEmpty)
+    .map { x =>
+      val value = x.parameterAssign.code.next().split("[.]").lastOption.getOrElse("")
+      (x.parameter.next(), value)
+    }
+    .filter { (_, value) =>
+      value.nonEmpty
+    }
+    .toList
+
   /** List of all parameters annotated with Spring's `Value` annotation, along with the property name.
     */
   private def annotatedParameters(): List[(MethodParameterIn, String)] = cpg.annotation

src/main/scala/ai/privado/languageEngine/java/tagger/sink/JavaAPITagger.scala

@@ -135,7 +135,10 @@ class JavaAPITagger(cpg: Cpg, ruleCache: RuleCache, privadoInputConfig: PrivadoI
         List()
     }
 
-    val markedAPISinks = cpg.call.where(_.tag.nameExact(InternalTag.API_SINK_MARKED.toString)).l
+    val markedAPISinks = cpg.call
+      .where(_.tag.nameExact(InternalTag.API_SINK_MARKED.toString))
+      .whereNot(_.tag.nameExact(InternalTag.API_URL_MARKED.toString))
+      .l
 
     apiTaggerToUse match {
       case APITaggerVersionJava.V1Tagger =>

src/main/scala/ai/privado/languageEngine/java/tagger/sink/api/JavaAPISinkEndpointMapperByNonInitMethod.scala

@@ -0,0 +1,109 @@
+package ai.privado.languageEngine.java.tagger.sink.api
+
+import ai.privado.cache.RuleCache
+import ai.privado.model.{Constants, InternalTag, NodeType, RuleInfo}
+import ai.privado.tagger.PrivadoParallelCpgPass
+import io.shiftleft.codepropertygraph.generated.Cpg
+import io.shiftleft.semanticcpg.language.*
+import ai.privado.languageEngine.java.language.*
+import ai.privado.tagger.utility.APITaggerUtility.{
+  getLiteralCode,
+  resolveDomainFromSource,
+  tagAPIWithDomainAndUpdateRuleCache
+}
+import ai.privado.utility.Utilities.{addRuleTags, getDomainFromString, storeForTag}
+import io.shiftleft.codepropertygraph.generated.nodes.AstNode
+
+class JavaAPISinkEndpointMapperByNonInitMethod(cpg: Cpg, ruleCache: RuleCache)
+    extends PrivadoParallelCpgPass[String](cpg) {
+
+  private val methodFullNameSplitter = "[:(]"
+
+  private val apiMatchingRegex =
+    ruleCache.getAllRuleInfo.filter(_.nodeType == NodeType.API).map(_.combinedRulePattern).mkString("(", "|", ")")
+
+  private val thirdPartyRuleInfo = ruleCache.getRuleInfo(Constants.thirdPartiesAPIRuleId)
+  override def generateParts(): Array[String] = {
+
+    /* General assumption - there is a function which creates a client, and the usage of the client and binding
+     happens via dependency injection which can very according to the framework used.
+     If we identify such a function and can point to the usage of a particular endpoint in it,
+     we can say the client uses the following endpoint
+     */
+
+    if (thirdPartyRuleInfo.isDefined) {
+      cpg.call
+        .where(_.tag.nameExact(InternalTag.API_SINK_MARKED.toString))
+        .methodFullName
+        .map(_.split(methodFullNameSplitter).headOption.getOrElse(""))
+        .filter(_.nonEmpty)
+        .map { methodNamespace =>
+          val parts = methodNamespace.split("[.]")
+          if parts.nonEmpty then parts.dropRight(1).mkString(".") else ""
+        }
+        .dedup
+        .toArray
+    } else
+      Array[String]()
+  }
+
+  override def runOnPart(builder: DiffGraphBuilder, typeFullName: String): Unit = {
+
+    cpg.method.signature(s"$typeFullName$methodFullNameSplitter.*").foreach { clientReturningMethod =>
+      val matchingProperties = clientReturningMethod.ast.originalProperty.value(apiMatchingRegex).dedup.l
+
+      val impactedApiCalls = cpg.call
+        .methodFullName(s"$typeFullName.*")
+        .where(_.tag.nameExact(InternalTag.API_SINK_MARKED.toString))
+        .l
+
+      if (matchingProperties.nonEmpty) {
+        matchingProperties.foreach { propertyNode =>
+          val domain = getDomainFromString(propertyNode.value)
+          impactedApiCalls.foreach { apiCall =>
+            tagAPIWithDomainAndUpdateRuleCache(
+              builder,
+              thirdPartyRuleInfo.get,
+              ruleCache,
+              domain,
+              apiCall,
+              propertyNode
+            )
+            storeForTag(builder, apiCall, ruleCache)(InternalTag.API_URL_MARKED.toString)
+          }
+        }
+      } else { // There is no property node available to be used, try with parameter
+        val variableRegex      = ruleCache.getSystemConfigByKey(Constants.apiIdentifier)
+        val matchingParameters = clientReturningMethod.parameter.name(variableRegex).l
+
+        if (matchingParameters.nonEmpty) {
+          matchingParameters.foreach { parameter =>
+            val domain = resolveDomainFromSource(parameter)
+            impactedApiCalls.foreach { apiCall =>
+              tagAPIWithDomainAndUpdateRuleCache(builder, thirdPartyRuleInfo.get, ruleCache, domain, apiCall, parameter)
+              storeForTag(builder, apiCall, ruleCache)(InternalTag.API_URL_MARKED.toString)
+            }
+          }
+        } else { // There is no matching parameter to be used, try with identifier
+          val matchingIdentifiers = clientReturningMethod.ast.isIdentifier.name(variableRegex).l
+          if (matchingIdentifiers.nonEmpty) {
+            matchingIdentifiers.foreach { identifier =>
+              val domain = resolveDomainFromSource(identifier)
+              impactedApiCalls.foreach { apiCall =>
+                tagAPIWithDomainAndUpdateRuleCache(
+                  builder,
+                  thirdPartyRuleInfo.get,
+                  ruleCache,
+                  domain,
+                  apiCall,
+                  identifier
+                )
+                storeForTag(builder, apiCall, ruleCache)(InternalTag.API_URL_MARKED.toString)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

src/main/scala/ai/privado/languageEngine/java/tagger/sink/api/JavaAPISinkTagger.scala

@@ -17,6 +17,9 @@ object JavaAPISinkTagger extends APISinkTagger {
       new JavaAPISinkByParameterTagger(cpg, ruleCache).createAndApply()
 
     new JavaAPISinkByMethodFullNameTagger(cpg, ruleCache).createAndApply()
+
+    // Invoke API Endpoint mappers
+    new JavaAPISinkEndpointMapperByNonInitMethod(cpg, ruleCache).createAndApply()
   }
 
 }

src/main/scala/ai/privado/model/PrivadoTag.scala

@@ -47,6 +47,7 @@ object InternalTag extends Enumeration {
 
   // API Tags
   val API_SINK_MARKED = Value("API_SINK_MARKED")
+  val API_URL_MARKED  = Value("API_URL_MARKED")
 
   lazy val valuesAsString = InternalTag.values.map(value => value.toString())
 

src/main/scala/ai/privado/tagger/utility/APITaggerUtility.scala

@@ -38,8 +38,9 @@ import ai.privado.utility.Utilities.{
 }
 import io.joern.dataflowengineoss.language.*
 import io.joern.dataflowengineoss.queryengine.{EngineConfig, EngineContext}
-import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, CfgNode, Member}
+import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, CfgNode, JavaProperty, Member}
 import overflowdb.BatchedUpdate
+import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 object APITaggerUtility {
 
@@ -48,8 +49,9 @@ object APITaggerUtility {
 
   def getLiteralCode(element: AstNode): String = {
     val literalCode = element match {
-      case member: Member => member.name
-      case _              => element.code.split(" ").last
+      case member: Member             => member.name
+      case propertyNode: JavaProperty => propertyNode.value
+      case _                          => element.code.split(" ").last
     }
 
     element.originalPropertyValue.getOrElse(literalCode)
@@ -110,12 +112,27 @@ object APITaggerUtility {
     urlValue.stripPrefix("\"").stripSuffix("\"")
   }
 
-  private def resolveDomainFromSource(sourceNode: AstNode): String = {
+  def resolveDomainFromSource(sourceNode: AstNode): String = {
     val sourceDomain = sourceNode.originalPropertyValue.getOrElse(getLiteralCode(sourceNode))
     if (sourceDomain.matches(SERVICE_URL_REGEX_PATTERN)) {
       sourceDomain.split("//").last
     } else {
       getDomainFromString(sourceDomain)
     }
   }
+
+  def tagAPIWithDomainAndUpdateRuleCache(
+    builder: DiffGraphBuilder,
+    ruleInfo: RuleInfo,
+    ruleCache: RuleCache,
+    domain: String,
+    apiNode: AstNode,
+    apiUrlNode: AstNode
+  ) = {
+    val newRuleIdToUse = ruleInfo.id + "." + domain
+    ruleCache.setRuleInfo(ruleInfo.copy(id = newRuleIdToUse, name = ruleInfo.name + " " + domain))
+    addRuleTags(builder, apiNode, ruleInfo, ruleCache, Some(newRuleIdToUse))
+    storeForTag(builder, apiNode, ruleCache)(Constants.apiUrl + newRuleIdToUse, getLiteralCode(apiUrlNode))
+
+  }
 }