Merge pull request #572 from HXSecurity/develop

Release v1.13.0

.github/workflows/code-check.yml

@@ -157,24 +157,18 @@ jobs:
             ossutil cp -rf dongtai-agent/src/main/resources/bin/agent_latest.tar.gz oss://dongtai-helm-charts/agent_${{ steps.version.outputs.GITHUB_REF }}/java/latest/ --meta x-oss-object-acl:public-read
           fi
 
-      - name: Set the value
-        id: release
-        run: |
-          if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV; echo "helm_mysql=test" >> $GITHUB_ENV 
-          elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV; echo "helm_mysql=beta" >> $GITHUB_ENV 
-          else echo "helm_ns=main" >> $GITHUB_ENV ; echo "helm_mysql=temp" >> $GITHUB_ENV ;fi
+      - name: deploy to cluster A
+        uses: tscuite/kubectl-helm-action@main
+        env:
+          MAX: false
+          PROJECT: agent
+          TOKEN_SCA: ${{ secrets.TOKEN_SCA }}
+          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
 
-      - name: deploy to cluster
-        uses: wahyd4/kubectl-helm-action@master
+      - name: deploy to cluster B
+        uses: tscuite/kubectl-helm-action@main
         env:
+          MAX: true
+          PROJECT: agent
+          TOKEN_SCA: ${{ secrets.MAX_TOKEN_SCA }}
           KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
-        with:
-          args: |
-            git clone https://github.com/HXSecurity/DongTai.git
-            helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
-            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}.huoxian.cn \
-            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.agent_number=iast${{github.run_number}} --set develop.agentZip=${{ env.helm_ns }} --values https://charts.dongtai.io/devops.yaml
-            helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }}-max ./DongTai/deploy/kubernetes/helm/ \
-            --set max=true --set sca.sca_token=${{ secrets.MAX_TOKEN_SCA }} --set usb.usb_token=${{ secrets.MAX_TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}-max.huoxian.cn \
-            --set tag=max-${{ steps.version.outputs.GITHUB_REF }}-latest --set develop.agentZip=${{ env.helm_ns }} \
-            --set build.agent_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml

dongtai-agent/src/main/java/io/dongtai/iast/agent/IastProperties.java

@@ -17,7 +17,6 @@
 public class IastProperties {
     public final static Map<String, String> ATTACH_ARG_MAP = new HashMap<String, String>() {{
         put("debug", PropertyConstant.PROPERTY_DEBUG);
-        put("app_create", PropertyConstant.PROPERTY_APP_CREATE);
         put("app_name", PropertyConstant.PROPERTY_APP_NAME);
         put("app_version", PropertyConstant.PROPERTY_APP_VERSION);
         put("app_template", PropertyConstant.PROPERTY_APP_TEMPLATE);
@@ -129,20 +128,6 @@ public boolean isDebug() {
         return "true".equalsIgnoreCase(getDebugFlag());
     }
 
-    public Integer isAutoCreateProject() {
-        if (null == isAutoCreateProject) {
-            String result = System.getProperty(PropertyConstant.PROPERTY_APP_CREATE,
-                    System.getProperty("project.create", cfg.getProperty("project.create", "false"))
-            );
-            if ("true".equalsIgnoreCase(result)) {
-                isAutoCreateProject = 1;
-            } else {
-                isAutoCreateProject = 0;
-            }
-        }
-        return isAutoCreateProject;
-    }
-
     public String getProjectName() {
         if (null == projectName) {
             String[] names = new String[]{

dongtai-agent/src/main/java/io/dongtai/iast/agent/report/AgentRegisterReport.java

@@ -65,7 +65,6 @@ private String generateAgentRegisterMsg() {
         object.put("serverPath", ServerDetect.getWebServerPath());
         object.put("serverAddr", "");
         object.put("serverPort", "");
-        object.put("autoCreateProject", IastProperties.getInstance().isAutoCreateProject());
         object.put("projectVersion", IastProperties.getInstance().getProjectVersion());
         object.put("projectTemplateId", IastProperties.getInstance().getProjectTemplate());
 

dongtai-common/src/main/java/io/dongtai/iast/common/config/ConfigBuilder.java

@@ -24,6 +24,8 @@ private ConfigBuilder() {
                 Config.<Boolean>create(ConfigKey.ENABLE_LOGGER));
         this.configMap.put(ConfigKey.LOGGER_LEVEL,
                 Config.<String>create(ConfigKey.LOGGER_LEVEL));
+        this.configMap.put(ConfigKey.VALIDATED_SINK,
+                Config.<Boolean>create(ConfigKey.VALIDATED_SINK).setDefaultValue(false));
     }
 
     public static ConfigBuilder getInstance() {
@@ -62,6 +64,7 @@ public void update(JSONObject config) {
         updateString(config, ConfigKey.JsonKey.JSON_VERSION_HEADER_KEY);
         updateBool(config, ConfigKey.JsonKey.JSON_ENABLE_LOGGER);
         updateString(config, ConfigKey.JsonKey.JSON_LOGGER_LEVEL);
+        updateBool(config, ConfigKey.JsonKey.JSON_VALIDATED_SINK);
         updateRequestDenyList(config);
     }
 

dongtai-common/src/main/java/io/dongtai/iast/common/config/ConfigKey.java

@@ -8,6 +8,7 @@ public enum ConfigKey {
     VERSION_HEADER_KEY,
     ENABLE_LOGGER,
     LOGGER_LEVEL,
+    VALIDATED_SINK,
     ;
 
     public enum JsonKey {
@@ -18,6 +19,7 @@ public enum JsonKey {
         JSON_VERSION_HEADER_KEY("version_header_name", VERSION_HEADER_KEY),
         JSON_ENABLE_LOGGER("enable_log", ENABLE_LOGGER),
         JSON_LOGGER_LEVEL("log_level", LOGGER_LEVEL),
+        JSON_VALIDATED_SINK("report_validated_sink", VALIDATED_SINK),
         ;
 
         private final String key;

dongtai-common/src/main/java/io/dongtai/iast/common/constants/AgentConstant.java

@@ -1,7 +1,7 @@
 package io.dongtai.iast.common.constants;
 
 public class AgentConstant {
-    public static final String VERSION_VALUE = "v1.12.0";
+    public static final String VERSION_VALUE = "v1.13.0";
     public static final String LANGUAGE = "JAVA";
     public static final String THREAD_NAME_PREFIX = "DongTai-IAST-";
     public static final String THREAD_NAME_PREFIX_CORE = "DongTai-IAST-Core-";

dongtai-common/src/main/java/io/dongtai/iast/common/constants/PropertyConstant.java

@@ -2,7 +2,6 @@
 
 public class PropertyConstant {
     public static final String PROPERTY_DEBUG = "dongtai.debug";
-    public static final String PROPERTY_APP_CREATE = "dongtai.app.create";
     public static final String PROPERTY_APP_NAME = "dongtai.app.name";
     public static final String PROPERTY_APP_VERSION = "dongtai.app.version";
     public static final String PROPERTY_APP_TEMPLATE = "dongtai.app.template";
@@ -32,5 +31,5 @@ public class PropertyConstant {
     public static final String PROPERTY_POLICY_PATH = "dongtai.policy.path";
     public static final String PROPERTY_UUID_PATH = "dongtai.uuid.path";
     public static final String PROPERTY_DISABLED_PLUGINS = "dongtai.disabled.plugins";
-    public static final String PROPERTY_DISABLED_FEATURES = "dongtai.disabled_features";
+    public static final String PROPERTY_DISABLED_FEATURES = "dongtai.disabled.features";
 }

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/IastClassFileTransformer.java

@@ -62,6 +62,13 @@ public static IastClassFileTransformer getInstance(Instrumentation inst, PolicyM
         return INSTANCE;
     }
 
+    public static IastClassFileTransformer getInstance() {
+        if (null != INSTANCE) {
+            return INSTANCE;
+        }
+        return null;
+    }
+
     IastClassFileTransformer(Instrumentation inst, PolicyManager policyManager) {
         this.inst = inst;
         this.isDumpClass = EngineManager.getInstance().isEnableDumpClass();
@@ -112,29 +119,34 @@ public byte[] transform(final ClassLoader loader,
                             final Class<?> classBeingRedefined,
                             final ProtectionDomain protectionDomain,
                             final byte[] srcByteCodeArray) {
+        String threadName = Thread.currentThread().getName();
+        if (threadName.startsWith("DongTai-IAST-Core")) {
+            return null;
+        }
+
+        if (internalClassName == null
+                || internalClassName.startsWith("io/dongtai/")
+                || internalClassName.startsWith("com/secnium/iast/")
+                || internalClassName.startsWith("java/lang/iast/")
+                || internalClassName.startsWith("cn/huoxian/iast/")
+                || internalClassName.startsWith("META-INF/")
+                || "module-info".equals(internalClassName)) {
+            return null;
+        }
+
+        if (null != loader && loader.toString().toLowerCase().contains("rasp")) {
+            return null;
+        }
+
         try {
             ScopeManager.SCOPE_TRACKER.getPolicyScope().enterAgent();
 
-            if (internalClassName == null
-                    || internalClassName.startsWith("io/dongtai/")
-                    || internalClassName.startsWith("com/secnium/iast/")
-                    || internalClassName.startsWith("java/lang/iast/")
-                    || internalClassName.startsWith("cn/huoxian/iast/")
-                    || internalClassName.startsWith("META-INF/")
-                    || "module-info".equals(internalClassName)) {
-                return null;
-            }
-
             if (" com/alibaba/fastjson/JSON".substring(1).equals(internalClassName)) {
                 FastjsonCheck.setJsonClassLoader(loader);
             } else if (" com/alibaba/fastjson/parser/ParserConfig".substring(1).equals(internalClassName)) {
                 FastjsonCheck.setParseConfigClassLoader(loader);
             }
 
-            if (null != loader && loader.toString().toLowerCase().contains("rasp")) {
-                return null;
-            }
-
             if (loader != null && protectionDomain != null) {
                 final CodeSource codeSource = protectionDomain.getCodeSource();
                 if (codeSource == null) {
@@ -156,7 +168,6 @@ public byte[] transform(final ClassLoader loader,
 
             ClassContext classContext = new ClassContext(cr, loader);
             if (Modifier.isInterface(classContext.getModifier())) {
-                sourceCodeBak = null;
                 return null;
             }
             final String className = classContext.getClassName();
@@ -186,11 +197,9 @@ public byte[] transform(final ClassLoader loader,
                     return dumpClassIfNecessary(cr.getClassName(), cw.toByteArray(), srcByteCodeArray);
                 }
             }
-            sourceCodeBak = null;
         } catch (Throwable throwable) {
             DongTaiLog.warn(ErrorCode.get("TRANSFORM_CLASS_FAILED"), internalClassName, throwable);
         } finally {
-            classDiagram.setLoader(null);
             ScopeManager.SCOPE_TRACKER.getPolicyScope().leaveAgent();
         }
 
@@ -347,5 +356,9 @@ public void reTransform() {
     public static HashMap<Object, byte[]> getTransformMap() {
         return transformMap;
     }
+
+    public IastClassDiagram getClassDiagram() {
+        return classDiagram;
+    }
 }
 

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/asm/AsmMethods.java

@@ -219,6 +219,12 @@ static Method getAsmMethod(final Class<?> clazz,
             SpyDispatcher.class,
             "isFirstLevelSink"
     );
+
+    Method SPY$enterValidator = InnerHelper.getAsmMethod(
+            SpyDispatcher.class,
+            "enterValidator"
+    );
+
     Method SPY$enterIgnoreInternal = InnerHelper.getAsmMethod(
             SpyDispatcher.class,
             "enterIgnoreInternal"

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/core/DispatchClassPlugin.java

@@ -54,6 +54,7 @@ public class ClassVisit extends AbstractClassVisitor {
                     new SourceAdapter(),
                     new PropagatorAdapter(),
                     new SinkAdapter(),
+                    new ValidatorAdapter(),
             };
         }
 

dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/core/adapter/ValidatorAdapter.java

@@ -0,0 +1,54 @@
+package io.dongtai.iast.core.bytecode.enhance.plugin.core.adapter;
+
+import io.dongtai.iast.core.bytecode.enhance.MethodContext;
+import io.dongtai.iast.core.handler.hookpoint.models.policy.PolicyNode;
+import io.dongtai.iast.core.handler.hookpoint.models.policy.ValidatorNode;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+import java.util.Set;
+
+public class ValidatorAdapter extends MethodAdapter {
+    /**
+     * @param adapter
+     * @param mv
+     * @param context
+     * @param policyNodes
+     */
+    @Override
+    public void onMethodEnter(MethodAdviceAdapter adapter, MethodVisitor mv, MethodContext context, Set<PolicyNode> policyNodes) {
+    }
+
+    /**
+     * @param adapter
+     * @param mv
+     * @param opcode
+     * @param context
+     * @param policyNodes
+     */
+    @Override
+    public void onMethodExit(MethodAdviceAdapter adapter, MethodVisitor mv, int opcode, MethodContext context, Set<PolicyNode> policyNodes) {
+        for (PolicyNode policyNode : policyNodes) {
+            if (!(policyNode instanceof ValidatorNode)) {
+                continue;
+            }
+
+            Label elseLabel = new Label();
+            Label endLabel = new Label();
+
+            isEnterScope(adapter);
+            mv.visitJumpInsn(Opcodes.IFEQ, elseLabel);
+
+            adapter.trackMethod(opcode, policyNode, true);
+
+            adapter.mark(elseLabel);
+            adapter.mark(endLabel);
+        }
+    }
+
+    private void isEnterScope(MethodAdviceAdapter adapter) {
+        adapter.invokeStatic(ASM_TYPE_SPY_HANDLER, SPY_HANDLER$getDispatcher);
+        adapter.invokeInterface(ASM_TYPE_SPY_DISPATCHER, SPY$enterValidator);
+    }
+}

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/SpyDispatcherImpl.java

@@ -394,7 +394,8 @@ public void collectDubboResponse(Object result, byte status) {
             }
 
             if (!ScopeManager.SCOPE_TRACKER.getScope(Scope.DUBBO_REQUEST).isFirst()
-                    || !ScopeManager.SCOPE_TRACKER.getScope(Scope.DUBBO_ENTRY).in()) {
+                    || !ScopeManager.SCOPE_TRACKER.getScope(Scope.DUBBO_ENTRY).in()
+                    || ScopeManager.SCOPE_TRACKER.getScope(Scope.HTTP_REQUEST).in()) {
                 return;
             }
 
@@ -558,6 +559,17 @@ public void leaveSink() {
         }
     }
 
+    /**
+     * mark for enter validator entry point
+     */
+    @Override
+    public boolean enterValidator() {
+        if (!EngineManager.isEngineRunning()) {
+            return false;
+        }
+        return !ScopeManager.SCOPE_TRACKER.inAgent() && ScopeManager.SCOPE_TRACKER.inEnterEntry();
+    }
+
     /**
      * Determines whether it is a layer 1 Sink entry
      *
@@ -674,6 +686,9 @@ public boolean collectMethod(Object instance, Object[] parameters, Object retObj
             } else if ((policyNode instanceof SinkNode)) {
                 SinkImpl.solveSink(event, (SinkNode) policyNode);
                 return true;
+            } else if ((policyNode instanceof ValidatorNode)) {
+                ValidatorImpl.solveValidator(event,(ValidatorNode)policyNode, INVOKE_ID_SEQUENCER);
+                return true;
             }
 
             return false;
@@ -731,7 +746,7 @@ public boolean traceDubboInvoke(Object instance, String url, Object invocation,
     @Override
     public boolean isSkipCollectDubbo(Object invocation) {
         if (BlackUrlBypass.isBlackUrl()) {
-            Method setAttachmentMethod = null;
+            Method setAttachmentMethod;
             try {
                 setAttachmentMethod = invocation.getClass().getMethod("setAttachment", String.class, String.class);
                 setAttachmentMethod.setAccessible(true);
@@ -746,7 +761,7 @@ public boolean isSkipCollectDubbo(Object invocation) {
     @Override
     public boolean isSkipCollectFeign(Object instance) {
         if (BlackUrlBypass.isBlackUrl()) {
-            Field metadataField = null;
+            Field metadataField;
             try {
                 metadataField = instance.getClass().getDeclaredField("metadata");
                 metadataField.setAccessible(true);

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/controller/impl/DubboImpl.java

@@ -8,6 +8,7 @@
 import io.dongtai.iast.core.handler.context.ContextManager;
 import io.dongtai.iast.core.handler.hookpoint.IastClassLoader;
 import io.dongtai.iast.core.handler.hookpoint.models.MethodEvent;
+import io.dongtai.iast.core.handler.hookpoint.models.policy.PolicyNodeType;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.SourceNode;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.TaintPosition;
 import io.dongtai.iast.core.handler.hookpoint.models.taint.range.TaintRange;
@@ -178,6 +179,7 @@ public static void collectDubboRequestSource(Object handler, Object invocation,
 
         int invokeId = invokeIdSequencer.getAndIncrement();
         event.setInvokeId(invokeId);
+        event.setPolicyType(PolicyNodeType.SOURCE.getName());
 
         event.source = true;
         event.setCallStacks(StackUtils.createCallStack(4));

dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/controller/impl/PropagatorImpl.java

@@ -2,6 +2,7 @@
 
 import io.dongtai.iast.core.EngineManager;
 import io.dongtai.iast.core.handler.hookpoint.models.MethodEvent;
+import io.dongtai.iast.core.handler.hookpoint.models.policy.PolicyNodeType;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.PropagatorNode;
 import io.dongtai.iast.core.handler.hookpoint.models.policy.TaintPosition;
 import io.dongtai.iast.core.handler.hookpoint.models.taint.range.*;
@@ -63,6 +64,7 @@ private static void addPropagator(PropagatorNode propagatorNode, MethodEvent eve
         event.setCallStacks(StackUtils.createCallStack(6));
         int invokeId = invokeIdSequencer.getAndIncrement();
         event.setInvokeId(invokeId);
+        event.setPolicyType(PolicyNodeType.PROPAGATOR.getName());
         EngineManager.TRACK_MAP.get().put(invokeId, event);
     }