Merge pull request #18341 from asgerf/py/diff-informed

Python: enable diff-informed data flow queries

python/ql/lib/semmle/python/security/dataflow/CleartextLoggingQuery.qll

@@ -21,6 +21,8 @@ private module CleartextLoggingConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "Clear-text logging of sensitive information" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/CleartextStorageQuery.qll

@@ -21,6 +21,8 @@ private module CleartextStorageConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "Clear-text storage of sensitive information" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/CodeInjectionQuery.qll

@@ -17,6 +17,8 @@ private module CodeInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "code injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/CommandInjectionQuery.qll

@@ -20,6 +20,8 @@ module CommandInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "command injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/CookieInjectionQuery.qll

@@ -20,6 +20,8 @@ module CookieInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "cookie injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/HttpHeaderInjectionQuery.qll

@@ -16,6 +16,8 @@ private module HeaderInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node node) { node instanceof HttpHeaderInjection::Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof HttpHeaderInjection::Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "HTTP Header injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/LdapInjectionQuery.qll

@@ -19,6 +19,8 @@ private module LdapInjectionDnConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof DnSink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof DnSanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "LDAP injection via the distinguished name (DN) parameter" vulnerabilities. */
@@ -30,6 +32,8 @@ private module LdapInjectionFilterConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof FilterSink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof FilterSanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "LDAP injection via the filter parameter" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/LogInjectionQuery.qll

@@ -17,6 +17,8 @@ private module LogInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "log injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/NoSqlInjectionQuery.qll

@@ -56,6 +56,8 @@ module NoSqlInjectionConfig implements DataFlow::StateConfigSig {
   predicate isBarrier(DataFlow::Node node) {
     node = any(NoSqlSanitizer noSqlSanitizer).getAnInput()
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 module NoSqlInjectionFlow = TaintTracking::GlobalWithState<NoSqlInjectionConfig>;

python/ql/lib/semmle/python/security/dataflow/PamAuthorizationQuery.qll

@@ -31,6 +31,8 @@ private module PamAuthorizationConfig implements DataFlow::ConfigSig {
     // Flow from handle to the authenticate call in the final step
     exists(VulnPamAuthCall c | c.getArg(0) = node1 | node2 = c)
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "PAM Authorization" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/PathInjectionQuery.qll

@@ -71,6 +71,8 @@ module PathInjectionConfig implements DataFlow::StateConfigSig {
     stateFrom instanceof NotNormalized and
     stateTo instanceof NormalizedUnchecked
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "path injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/PolynomialReDoSCustomizations.qll

@@ -35,6 +35,11 @@ module PolynomialReDoS {
     /** Gets the regex that is being executed by this node. */
     abstract RegExpTerm getRegExp();
 
+    /** Gets a term within the regexp that may perform polynomial back-tracking. */
+    final PolynomialBackTrackingTerm getABacktrackingTerm() {
+      result.getRootTerm() = this.getRegExp()
+    }
+
     /**
      * Gets the node to highlight in the alert message.
      */

python/ql/lib/semmle/python/security/dataflow/PolynomialReDoSQuery.qll

@@ -17,6 +17,14 @@ private module PolynomialReDoSConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSinkLocation(DataFlow::Node sink) {
+    result = sink.(Sink).getHighlight().getLocation()
+    or
+    result = sink.(Sink).getABacktrackingTerm().getLocation()
+  }
 }
 
 /** Global taint-tracking for detecting "polynomial regular expression denial of service (ReDoS)" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/ReflectedXssQuery.qll

@@ -17,6 +17,8 @@ private module ReflectedXssConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "reflected server-side cross-site scripting" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/RegexInjectionQuery.qll

@@ -18,6 +18,14 @@ private module RegexInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSinkLocation(DataFlow::Node sink) {
+    result = sink.(Sink).getLocation()
+    or
+    result = sink.(Sink).getRegexExecution().getLocation()
+  }
 }
 
 /** Global taint-tracking for detecting "regular expression injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/ServerSideRequestForgeryQuery.qll

@@ -29,6 +29,12 @@ private module FullServerSideRequestForgeryConfig implements DataFlow::ConfigSig
     or
     node instanceof FullUrlControlSanitizer
   }
+
+  predicate observeDiffInformedIncrementalMode() {
+    // The partial request forgery query depends on `fullyControlledRequest` to reject alerts about
+    // such full-controlled requests, regardless of the associated source.
+    none()
+  }
 }
 
 /**
@@ -58,6 +64,13 @@ private module PartialServerSideRequestForgeryConfig implements DataFlow::Config
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSinkLocation(DataFlow::Node sink) {
+    // Note: this query does not select the sink itself
+    result = sink.(Sink).getRequest().getLocation()
+  }
 }
 
 /**

python/ql/lib/semmle/python/security/dataflow/SqlInjectionQuery.qll

@@ -17,6 +17,8 @@ private module SqlInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "SQL injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/StackTraceExposureQuery.qll

@@ -26,6 +26,8 @@ private module StackTraceExposureConfig implements DataFlow::ConfigSig {
       nodeTo = attr
     )
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "stack trace exposure" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/TarSlipQuery.qll

@@ -17,6 +17,8 @@ private module TarSlipConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "tar slip" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/TemplateInjectionQuery.qll

@@ -17,6 +17,8 @@ private module TemplateInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node node) { node instanceof Sink }
 
   predicate isBarrierIn(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "template injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/UnsafeDeserializationQuery.qll

@@ -17,6 +17,8 @@ private module UnsafeDeserializationConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "code execution from deserialization" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/UnsafeShellCommandConstructionQuery.qll

@@ -28,6 +28,16 @@ module UnsafeShellCommandConstructionConfig implements DataFlow::ConfigSig {
 
   // override to require the path doesn't have unmatched return steps
   DataFlow::FlowFeature getAFeature() { result instanceof DataFlow::FeatureHasSourceCallContext }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSinkLocation(DataFlow::Node sink) {
+    result = sink.(Sink).getLocation()
+    or
+    result = sink.(Sink).getStringConstruction().getLocation()
+    or
+    result = sink.(Sink).getCommandExecution().getLocation()
+  }
 }
 
 /** Global taint-tracking for detecting "shell command constructed from library input" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/UrlRedirectQuery.qll

@@ -32,6 +32,8 @@ private module UrlRedirectConfig implements DataFlow::StateConfigSig {
   ) {
     any(UrlRedirect::AdditionalFlowStep a).step(nodeFrom, stateFrom, nodeTo, stateTo)
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "URL redirection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/WeakSensitiveDataHashingQuery.qll

@@ -33,6 +33,8 @@ module NormalHashFunction {
     predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
       sensitiveDataExtraStepForCalls(node1, node2)
     }
+
+    predicate observeDiffInformedIncrementalMode() { any() }
   }
 
   /** Global taint-tracking for detecting "use of a broken or weak cryptographic hashing algorithm on sensitive data" vulnerabilities. */
@@ -63,6 +65,8 @@ module ComputationallyExpensiveHashFunction {
     predicate isAdditionalFlowStep(DataFlow::Node node1, DataFlow::Node node2) {
       sensitiveDataExtraStepForCalls(node1, node2)
     }
+
+    predicate observeDiffInformedIncrementalMode() { any() }
   }
 
   /** Global taint-tracking for detecting "use of a broken or weak cryptographic hashing algorithm on passwords" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/XmlBombQuery.qll

@@ -17,6 +17,8 @@ private module XmlBombConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "XML bomb" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/XpathInjectionQuery.qll

@@ -17,6 +17,8 @@ private module XpathInjectionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "Xpath Injection" vulnerabilities. */

python/ql/lib/semmle/python/security/dataflow/XxeQuery.qll

@@ -17,6 +17,8 @@ private module XxeConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "XML External Entity (XXE)" vulnerabilities. */

python/ql/src/Security/CWE-020-ExternalAPIs/ExternalAPIs.qll

@@ -171,6 +171,10 @@ private module UntrustedDataToExternalApiConfig implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof ExternalApiDataNode }
+
+  predicate observeDiffInformedIncrementalMode() {
+    none() // Not used for PR analysis
+  }
 }
 
 /** Global taint-tracking from `RemoteFlowSource`s to `ExternalApiDataNode`s. */

python/ql/src/Security/CWE-327/FluentApiModel.qll

@@ -110,6 +110,10 @@ module InsecureContextConfiguration implements DataFlow::StateConfigSig {
       )
     )
   }
+
+  predicate observeDiffInformedIncrementalMode() {
+    none() // Too complicated, but might be possible after some refactoring.
+  }
 }
 
 private module InsecureContextFlow = DataFlow::GlobalWithState<InsecureContextConfiguration>;

python/ql/src/Security/CWE-730/PolynomialReDoS.ql

@@ -23,7 +23,7 @@ from
 where
   PolynomialReDoSFlow::flowPath(source, sink) and
   sinkNode = sink.getNode() and
-  regexp.getRootTerm() = sinkNode.getRegExp()
+  regexp = sinkNode.getABacktrackingTerm()
 //   not (
 //     source.getNode().(Source).getKind() = "url" and
 //     regexp.isAtEndLine()

python/ql/src/Security/CWE-798/HardcodedCredentials.ql

@@ -119,6 +119,8 @@ private module HardcodedCredentialsConfig implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { source instanceof HardcodedValueSource }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof CredentialSink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 module HardcodedCredentialsFlow = TaintTracking::Global<HardcodedCredentialsConfig>;

python/ql/src/experimental/Security/CWE-022bis/TarSlipImprov.ql

@@ -109,6 +109,8 @@ private module TarSlipImprovConfig implements DataFlow::ConfigSig {
     nodeFrom = nodeTo.(API::CallNode).getArg(0) and
     nodeFrom = tarfileOpen().getReturn().getAValueReachableFromSource()
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting more "TarSlip" vulnerabilities. */

python/ql/src/experimental/Security/CWE-091/XsltInjectionQuery.qll

@@ -19,6 +19,8 @@ module XsltInjectionConfig implements DataFlow::ConfigSig {
     // opted for the more simple approach.
     nodeTo = elementTreeConstruction(nodeFrom)
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 module XsltInjectionFlow = TaintTracking::Global<XsltInjectionConfig>;

python/ql/src/experimental/Security/CWE-094/Js2Py.ql

@@ -24,6 +24,8 @@ module Js2PyFlowConfig implements DataFlow::ConfigSig {
     API::moduleImport("js2py").getMember(["eval_js", "eval_js6", "EvalJs"]).getACall().getArg(_) =
       node
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 module Js2PyFlow = TaintTracking::Global<Js2PyFlowConfig>;

python/ql/src/experimental/Security/CWE-176/UnicodeBypassValidationQuery.qll

@@ -75,6 +75,8 @@ private module UnicodeBypassValidationConfig implements DataFlow::StateConfigSig
     ) and
     state instanceof PostValidation
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Global taint-tracking for detecting "Unicode transformation mishandling" vulnerabilities. */

python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHeaderValue/TimingAttackAgainstHeaderValue.ql

@@ -23,6 +23,8 @@ private module TimingAttackAgainstHeaderValueConfig implements DataFlow::ConfigS
   predicate isSource(DataFlow::Node source) { source instanceof ClientSuppliedSecret }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof CompareSink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 module TimingAttackAgainstHeaderValueFlow =

python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstSensitiveInfo/PossibleTimingAttackAgainstSensitiveInfo.ql

@@ -23,6 +23,8 @@ private module PossibleTimingAttackAgainstSensitiveInfoConfig implements DataFlo
   predicate isSource(DataFlow::Node source) { source instanceof SecretSource }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof NonConstantTimeComparisonSink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 module PossibleTimingAttackAgainstSensitiveInfoFlow =