Merge pull request #536 from HXSecurity/beta

Release v1.11.0

dongtai-agent/src/main/java/io/dongtai/iast/agent/Agent.java

@@ -39,6 +39,7 @@ private static String[] parseAgentArgs(String[] args) throws ParseException {
         attachOptions.addOption(build("log_path", "log_path", "optional: DongTai agent log print path."));
         attachOptions.addOption(build("log_disable_collector", "log_disable_collector", "optional: DongTai agent disable log collector."));
         attachOptions.addOption(build("disabled_plugins", "disabled_plugins", "optional: DongTai agent disable plugins."));
+        attachOptions.addOption(build("disabled_features", "disabled_features", "optional: DongTai agent disable features."));
 
         CommandLineParser parser = new DefaultParser();
         HelpFormatter formatter = new HelpFormatter();

dongtai-agent/src/main/java/io/dongtai/iast/agent/IastProperties.java

@@ -34,6 +34,7 @@ public class IastProperties {
         put("log_disable_collector", PropertyConstant.PROPERTY_LOG_DISABLE_COLLECTOR);
         put("uuid_path", PropertyConstant.PROPERTY_UUID_PATH);
         put("disabled_plugins", PropertyConstant.PROPERTY_DISABLED_PLUGINS);
+        put("disabled_features", PropertyConstant.PROPERTY_DISABLED_FEATURES);
     }};
 
     private static IastProperties instance;

dongtai-agent/src/main/java/io/dongtai/iast/agent/LogCollector.java

@@ -22,6 +22,15 @@ public static void extractFluent() {
         }
         try {
             if (!isMacOs() && !isWindows()) {
+                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
+                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
+                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
+                FileUtils.confReplace(FLUENT_FILE_CONF);
+
+                String multiParserFile = IastProperties.getInstance().getTmpDir() + "parsers_multiline.conf";
+                FileUtils.getResourceToFile("bin/parsers_multiline.conf", multiParserFile);
+                FileUtils.confReplace(multiParserFile);
+
                 FLUENT_FILE = IastProperties.getInstance().getTmpDir() + "fluent";
                 File f = new File(FLUENT_FILE);
                 if (f.exists()) {
@@ -34,10 +43,6 @@ public static void extractFluent() {
                     FileUtils.getResourceToFile("bin/fluent", FLUENT_FILE);
                 }
 
-                String agentId = String.valueOf(AgentRegisterReport.getAgentId());
-                FLUENT_FILE_CONF = IastProperties.getInstance().getTmpDir() + "fluent-" + agentId + ".conf";
-                FileUtils.getResourceToFile("bin/fluent.conf", FLUENT_FILE_CONF);
-                FileUtils.confReplace(FLUENT_FILE_CONF);
                 if (!(new File(FLUENT_FILE)).setExecutable(true)) {
                     DongTaiLog.warn(ErrorCode.FLUENT_SET_EXECUTABLE_FAILED, FLUENT_FILE);
                 }

dongtai-agent/src/main/java/io/dongtai/iast/agent/middlewarerecognition/dubbo/DubboService.java

@@ -13,7 +13,7 @@ public boolean isMatch(RuntimeMXBean paramRuntimeMXBean, ClassLoader loader) {
         } catch (Throwable ignored) {
         }
         try {
-            loader.loadClass("com.alibaba.dubbo.monitor.support.MonitorFilter");
+            loader.loadClass(" com.alibaba.dubbo.monitor.support.MonitorFilter".substring(1));
             return true;
         } catch (Throwable ignored) {
         }

dongtai-agent/src/main/resources/bin/fluent.conf

@@ -3,6 +3,7 @@
   Daemon       OFF
   Log_Level    error
   HTTP_Server  Off
+  parsers_file parsers_multiline.conf
 [INPUT]
   Name         tail
   Path         ${LOG_PATH}
@@ -13,6 +14,7 @@
   Buffer_Max_Size   16MB
   Skip_Long_Lines   On
   Read_from_Head    true
+  multiline.parser multiline-regex-test
 [FILTER]
   Name         record_modifier
   Match        *

dongtai-agent/src/main/resources/bin/parsers_multiline.conf

@@ -0,0 +1,6 @@
+[MULTILINE_PARSER]
+    name          multiline-regex-test
+    type          regex
+    flush_timeout 1000
+    rule      "start_state"   "/(\d+\d+\-\d+\-\d+ \d+\:\d+\:\d+)(.*)/"  "cont"
+    rule      "cont"          "/(^\s+at.*|^Caused.*|^\s+\.\.\..*)/"     "cont"

dongtai-common/src/main/java/io/dongtai/iast/common/constants/AgentConstant.java

@@ -1,7 +1,7 @@
 package io.dongtai.iast.common.constants;
 
 public class AgentConstant {
-    public static final String VERSION_VALUE = "v1.10.0";
+    public static final String VERSION_VALUE = "v1.11.0";
     public static final String LANGUAGE = "JAVA";
     public static final String THREAD_NAME_PREFIX = "DongTai-IAST-";
     public static final String THREAD_NAME_PREFIX_CORE = "DongTai-IAST-Core-";

dongtai-common/src/main/java/io/dongtai/iast/common/constants/PropertyConstant.java

@@ -25,12 +25,12 @@ public class PropertyConstant {
     public static final String PROPERTY_JAR_API_URL = "iast.jar.api.url";
     public static final String PROPERTY_LOG_ADDRESS = "dongtai.log.address";
     public static final String PROPERTY_LOG_PORT = "dongtai.log.port";
-    public static final String PROPERTY_FALLBACK_VERSION = "dongtai.fallback.version";
     public static final String PROPERTY_DUMP_CLASS_PATH = "iast.dump.class.path";
     public static final String PROPERTY_DUMP_CLASS_ENABLE = "iast.dump.class.enable";
     public static final String PROPERTY_SERVICE_HEARTBEAT_INTERVAL = "iast.service.heartbeat.interval";
     public static final String PROPERTY_RESPONSE_LENGTH = "dongtai.response.length";
     public static final String PROPERTY_POLICY_PATH = "dongtai.policy.path";
     public static final String PROPERTY_UUID_PATH = "dongtai.uuid.path";
     public static final String PROPERTY_DISABLED_PLUGINS = "dongtai.disabled.plugins";
+    public static final String PROPERTY_DISABLED_FEATURES = "dongtai.disabled_features";
 }

dongtai-core/src/main/java/io/dongtai/iast/core/EngineManager.java

@@ -136,16 +136,16 @@ public static void enterHttpEntry(Map<String, Object> requestMeta) {
         }
         REQUEST_CONTEXT.set(requestMeta);
         TRACK_MAP.set(new HashMap<Integer, MethodEvent>(1024));
-        TAINT_HASH_CODES.set(new HashSet<Integer>());
-        TAINT_RANGES_POOL.set(new HashMap<Integer, TaintRanges>());
+        TAINT_HASH_CODES.set(new HashSet<Long>());
+        TAINT_RANGES_POOL.set(new HashMap<Long, TaintRanges>());
         ScopeManager.SCOPE_TRACKER.getScope(Scope.HTTP_ENTRY).enter();
     }
 
     public static void enterDubboEntry(Map<String, Object> requestMeta) {
         REQUEST_CONTEXT.set(requestMeta);
         TRACK_MAP.set(new HashMap<Integer, MethodEvent>(1024));
-        TAINT_HASH_CODES.set(new HashSet<Integer>());
-        TAINT_RANGES_POOL.set(new HashMap<Integer, TaintRanges>());
+        TAINT_HASH_CODES.set(new HashSet<Long>());
+        TAINT_RANGES_POOL.set(new HashMap<Long, TaintRanges>());
         ScopeManager.SCOPE_TRACKER.getScope(Scope.DUBBO_ENTRY).enter();
     }
 }

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/PluginRegister.java

@@ -12,6 +12,7 @@
 import io.dongtai.iast.core.bytecode.enhance.plugin.spring.DispatchApiCollector;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.Policy;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.PolicyManager;
+import io.dongtai.iast.core.utils.PropertyUtils;
 import org.objectweb.asm.ClassVisitor;
 
 import java.util.*;
@@ -28,7 +29,7 @@ public class PluginRegister {
 
     public PluginRegister() {
         this.plugins = new ArrayList<>();
-        List<String> disabledPlugins = getdisabledPlugins();
+        List<String> disabledPlugins = PropertyUtils.getDisabledPlugins();
         List<DispatchPlugin> allPlugins = new ArrayList<>(Arrays.asList(
                 new DispatchApiCollector(),
                 new DispatchJ2ee(),
@@ -43,12 +44,6 @@ public PluginRegister() {
         this.plugins.add(new DispatchClassPlugin());
     }
 
-    private List<String> getdisabledPlugins() {
-        return Optional.ofNullable(System.getProperty("dongtai.disabled.plugins"))
-                .map(s -> Arrays.asList(s.split(",")))
-                .orElse(null);
-    }
-
     public ClassVisitor initial(ClassVisitor classVisitor, ClassContext context, PolicyManager policyManager) {
         Policy policy = policyManager.getPolicy();
         if (policy == null) {

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/controller/impl/DubboImpl.java

@@ -27,7 +27,7 @@ public static void solveDubboRequest(Object handler, Object channel, Object requ
                 put("requestURL", u.getScheme() + "://" + u.getAuthority() + u.getPath());
                 put("requestURI", u.getPath());
                 put("queryString", "");
-                put("method", "DUBOO");
+                put("method", "DUBBO");
                 put("protocol", "DUBBO");
                 put("scheme", u.getScheme());
                 put("contextPath", "");
@@ -43,15 +43,14 @@ public static void solveDubboRequest(Object handler, Object channel, Object requ
     }
 
 
-
     public static void collectDubboRequestSource(Object handler, Object invocation, String methodName,
                                                  Object[] arguments, Map<String, ?> headers,
                                                  String hookClass, String hookMethod, String hookSign,
                                                  AtomicInteger invokeIdSequencer) {
         if (arguments == null || arguments.length == 0) {
             return;
         }
-        Map <String, Object> requestMeta = EngineManager.REQUEST_CONTEXT.get();
+        Map<String, Object> requestMeta = EngineManager.REQUEST_CONTEXT.get();
         if (requestMeta == null) {
             return;
         }
@@ -70,7 +69,7 @@ public static void collectDubboRequestSource(Object handler, Object invocation,
         tgt.add(new TaintPosition("P1"));
 
         SourceNode sourceNode = new SourceNode(src, tgt, null);
-        TaintPoolUtils.trackObject(event, sourceNode, arguments, 0);
+        TaintPoolUtils.trackObject(event, sourceNode, arguments, 0, true);
 
         Map<String, String> sHeaders = new HashMap<String, String>();
         if (headers != null) {
@@ -102,7 +101,7 @@ public static void collectDubboRequestSource(Object handler, Object invocation,
 
         // for display taint range (full arguments value)
         String fv = event.parameterValues.get(0).getValue();
-        int hash = System.identityHashCode(fv);
+        long hash = TaintPoolUtils.toStringHash(fv.hashCode(),System.identityHashCode(fv));
         int len = TaintRangesBuilder.getLength(fv);
         TaintRanges tr = new TaintRanges(new TaintRange(0, len));
         event.targetRanges.add(0, new MethodEvent.MethodEventTargetRange(hash, tr));

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/controller/impl/PropagatorImpl.java

@@ -165,7 +165,7 @@ private static boolean setTarget(PropagatorNode propagatorNode, MethodEvent even
     }
 
     private static TaintRanges getTaintRanges(Object obj) {
-        int hash = System.identityHashCode(obj);
+        long hash = TaintPoolUtils.getStringHash(obj);
         TaintRanges tr = EngineManager.TAINT_RANGES_POOL.get(hash);
         if (tr == null) {
             tr = new TaintRanges();
@@ -209,7 +209,7 @@ private static void trackTaintRange(PropagatorNode propagatorNode, MethodEvent e
             }
         }
 
-        int tgtHash = 0;
+        long tgtHash = 0;
         Object tgt = null;
         Set<TaintPosition> targetLocs = propagatorNode.getTargets();
         // may have multiple targets?
@@ -218,17 +218,17 @@ private static void trackTaintRange(PropagatorNode propagatorNode, MethodEvent e
         }
         if (TaintPosition.hasObject(targetLocs)) {
             tgt = event.objectInstance;
-            tgtHash = System.identityHashCode(tgt);
+            tgtHash = TaintPoolUtils.getStringHash(tgt);
             oldTaintRanges = getTaintRanges(tgt);
         } else if (TaintPosition.hasReturn(targetLocs)) {
             tgt = event.returnInstance;
-            tgtHash = System.identityHashCode(tgt);
+            tgtHash = TaintPoolUtils.getStringHash(tgt);
         } else if (TaintPosition.hasParameter(targetLocs)) {
             for (TaintPosition targetLoc : targetLocs) {
                 int parameterIndex = targetLoc.getParameterIndex();
                 if (event.parameterInstances.length > parameterIndex) {
                     tgt = event.parameterInstances[parameterIndex];
-                    tgtHash = System.identityHashCode(tgt);
+                    tgtHash = TaintPoolUtils.getStringHash(tgt);
                     oldTaintRanges = getTaintRanges(tgt);
                 }
             }

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/controller/impl/SourceImpl.java

@@ -81,26 +81,10 @@ private static boolean trackTarget(MethodEvent event, SourceNode sourceNode) {
             return false;
         }
 
-        TaintPoolUtils.trackObject(event, sourceNode, event.returnInstance, 0);
-        // @TODO: hook json serializer for custom model
-        handlerCustomModel(event, sourceNode);
+        TaintPoolUtils.trackObject(event, sourceNode, event.returnInstance, 0, false);
         return true;
     }
 
-    /**
-     * todo: 处理过程和结果需要细化
-     *
-     * @param event MethodEvent
-     */
-    public static void handlerCustomModel(MethodEvent event, SourceNode sourceNode) {
-        if (!"getSession".equals(event.getMethodName())) {
-            Set<Object> modelValues = TaintPoolUtils.parseCustomModel(event.returnInstance);
-            for (Object modelValue : modelValues) {
-                TaintPoolUtils.trackObject(event, sourceNode, modelValue, 0);
-            }
-        }
-    }
-
     private static boolean allowCall(MethodEvent event) {
         boolean allowed = true;
         if (METHOD_OF_GETATTRIBUTE.equals(event.getMethodName())) {

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/models/MethodEvent.java

@@ -82,9 +82,9 @@ public class MethodEvent {
      */
     public String returnValue;
 
-    private final Set<Integer> sourceHashes = new HashSet<Integer>();
+    private final Set<Long> sourceHashes = new HashSet<Long>();
 
-    private final Set<Integer> targetHashes = new HashSet<Integer>();
+    private final Set<Long> targetHashes = new HashSet<Long>();
 
     public List<MethodEventTargetRange> targetRanges = new ArrayList<MethodEventTargetRange>();
 
@@ -118,10 +118,10 @@ public JSONObject toJson() {
     }
 
     public static class MethodEventSourceType {
-        private final Integer hash;
+        private final Long hash;
         private final String type;
 
-        public MethodEventSourceType(Integer hash, String type) {
+        public MethodEventSourceType(Long hash, String type) {
             this.hash = hash;
             this.type = type;
         }
@@ -135,10 +135,10 @@ public JSONObject toJson() {
     }
 
     public static class MethodEventTargetRange {
-        private final Integer hash;
+        private final Long hash;
         private final TaintRanges ranges;
 
-        public MethodEventTargetRange(Integer hash, TaintRanges ranges) {
+        public MethodEventTargetRange(Long hash, TaintRanges ranges) {
             this.hash = hash;
             this.ranges = ranges;
         }
@@ -234,19 +234,19 @@ private String formatValue(Object val, boolean hasTaint) {
                 + (hasTaint ? "*" : "") + String.valueOf(str.length());
     }
 
-    public Set<Integer> getSourceHashes() {
+    public Set<Long> getSourceHashes() {
         return sourceHashes;
     }
 
-    public void addSourceHash(int hashcode) {
+    public void addSourceHash(long hashcode) {
         this.sourceHashes.add(hashcode);
     }
 
-    public Set<Integer> getTargetHashes() {
+    public Set<Long> getTargetHashes() {
         return targetHashes;
     }
 
-    public void addTargetHash(int hashCode) {
+    public void addTargetHash(long hashCode) {
         this.targetHashes.add(hashCode);
     }
 

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/models/taint/range/TaintRanges.java

@@ -58,6 +58,9 @@ public void untag(String[] untags) {
     }
 
     public boolean hasRequiredTaintTags(TaintTag[] tags) {
+        if (tags == null) {
+            return true;
+        }
         int total = tags.length;
         Map<String, Boolean> found = new HashMap<String, Boolean>();
         for (TaintTag tag : tags) {
@@ -71,6 +74,9 @@ public boolean hasRequiredTaintTags(TaintTag[] tags) {
     }
 
     public boolean hasDisallowedTaintTags(TaintTag[] tags) {
+        if (tags == null) {
+            return false;
+        }
         for (TaintTag tag : tags) {
             for (TaintRange taintRange : this.taintRanges) {
                 if (tag.equals(taintRange.getName())) {

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/models/taint/tag/TaintTag.java

@@ -27,9 +27,13 @@ public enum TaintTag {
     FTL_DECODED("ftl-decoded"),
     CSS_ENCODED("css-encoded"),
     XPATH_ENCODED("xpath-encoded"),
+    XPATH_DECODED("xpath-decoded"),
     LDAP_ENCODED("ldap-encoded"),
+    LDAP_DECODED("ldap-decoded"),
     OS_ENCODED("os-encoded"),
     VBSCRIPT_ENCODED("vbscript-encoded"),
+    HTTP_TOKEN_LIMITED_CHARS("http-token-limited-chars"),
+    NUMERIC_LIMITED_CHARS("numeric-limited-chars"),
     ;
 
     private final String key;

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/service/trace/DubboService.java

@@ -15,7 +15,7 @@ public class DubboService {
     public static void solveSyncInvoke(MethodEvent event, Object invocation, String url, Map<String, String> headers,
                                        AtomicInteger invokeIdSequencer) {
         try {
-            TaintPoolUtils.trackObject(event, null, event.parameterInstances, 0);
+            TaintPoolUtils.trackObject(event, null, event.parameterInstances, 0, false);
             boolean hasTaint = false;
             int sourceLen = 0;
             if (!event.getSourceHashes().isEmpty()) {
@@ -26,7 +26,7 @@ public static void solveSyncInvoke(MethodEvent event, Object invocation, String
 
             if (headers != null && headers.size() > 0) {
                 hasTaint = false;
-                TaintPoolUtils.trackObject(event, null, headers, 0);
+                TaintPoolUtils.trackObject(event, null, headers, 0, false);
                 if (event.getSourceHashes().size() > sourceLen) {
                     hasTaint = true;
                 }

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/service/trace/FeignService.java

@@ -27,7 +27,7 @@ public static void solveSyncInvoke(MethodEvent event, AtomicInteger invokeIdSequ
 
             // get args
             Object args = event.parameterInstances[0];
-            TaintPoolUtils.trackObject(event, null, args, 0);
+            TaintPoolUtils.trackObject(event, null, args, 0, true);
 
             boolean hasTaint = false;
             if (!event.getSourceHashes().isEmpty()) {

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/vulscan/VulnType.java

@@ -15,6 +15,14 @@ public enum VulnType {
     CRYPTO_BAD_MAC("crypto-bad-mac", "high", false),
     COOKIE_FLAGS_MISSING("cookie-flags-missing", "high", true),
     REFLECTED_XSS("reflected-xss", "medium", true),
+    SQL_INJECTION("sql-injection", "high", true),
+    HQL_INJECTION("hql-injection", "high", true),
+    LDAP_INJECTION("ldap-injection", "high", true),
+    CMD_INJECTION("cmd-injection", "high", true),
+    XPATH_INJECTION("xpath-injection", "high", true),
+    PATH_TRAVERSAL("path-traversal", "high", true),
+    XXE("xxe", "medium", true),
+    UNVALIDATED_REDIRECT("unvalidated-redirect", "low", true),
     ;
 
     public String getName() {