Merge pull request #560 from lostsnow/beta

bump version to 1.12.0
HXSecurity · Jul 17, 2023 · f3b28d3 · f3b28d3
2 parents 5b31d4c + 82ae9d0
commit f3b28d3
Show file tree

Hide file tree

Showing 33 changed files with 673 additions and 72 deletions.
diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml
@@ -148,17 +148,33 @@ jobs:
           if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then 
             ossutil cp -rf dongtai-agent/target/dongtai-agent.jar oss://dongtai-helm-charts/agent_test/java/latest/dongtai-agent.jar --meta x-oss-object-acl:public-read
             ossutil cp -rf dongtai-agent/src/main/resources/bin/ oss://dongtai-helm-charts/agent_test/java/latest/ --meta x-oss-object-acl:public-read
-          else  
+            cp dongtai-agent/target/dongtai-agent.jar dongtai-agent/src/main/resources/bin/ && cd dongtai-agent/src/main/resources/bin/ && tar -zcvf agent_latest.tar.gz * && cd ../../../../../
+            ossutil cp -rf dongtai-agent/src/main/resources/bin/agent_latest.tar.gz oss://dongtai-helm-charts/agent_test/java/latest/ --meta x-oss-object-acl:public-read
+          else
             ossutil cp -rf dongtai-agent/target/dongtai-agent.jar oss://dongtai-helm-charts/agent_${{ steps.version.outputs.GITHUB_REF }}/java/latest/dongtai-agent.jar --meta x-oss-object-acl:public-read
             ossutil cp -rf dongtai-agent/src/main/resources/bin/ oss://dongtai-helm-charts/agent_${{ steps.version.outputs.GITHUB_REF }}/java/latest/ --meta x-oss-object-acl:public-read
+            cp dongtai-agent/target/dongtai-agent.jar dongtai-agent/src/main/resources/bin/ && cd dongtai-agent/src/main/resources/bin/ && tar -zcvf agent_latest.tar.gz * && cd ../../../../../
+            ossutil cp -rf dongtai-agent/src/main/resources/bin/agent_latest.tar.gz oss://dongtai-helm-charts/agent_${{ steps.version.outputs.GITHUB_REF }}/java/latest/ --meta x-oss-object-acl:public-read
           fi
-          # ossutil cp -rf dongtai-agent/target/dongtai-agent.jar oss://dongtai-helm-charts/agent_test/java/dongtai-agent.jar --meta x-oss-object-acl:public-read
-          # ossutil cp -rf dongtai-agent/src/main/resources/bin/ oss://dongtai-helm-charts/agent_test/java/ --meta x-oss-object-acl:public-read
-
-      # - name: Trigger Openapi Workflow
-      #   uses: benc-uk/workflow-dispatch@v1
-      #   with:
-      #     workflow: Deploy DongTai Server To Dev
-      #     token: ${{ secrets.BIDAYA0_PAT_FOR_OPENAPI }}
-      #     ref: main
-      #     repo: HXSecurity/DongTai
+
+      - name: Set the value
+        id: release
+        run: |
+          if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV; echo "helm_mysql=test" >> $GITHUB_ENV 
+          elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV; echo "helm_mysql=beta" >> $GITHUB_ENV 
+          else echo "helm_ns=main" >> $GITHUB_ENV ; echo "helm_mysql=temp" >> $GITHUB_ENV ;fi
+
+      - name: deploy to cluster
+        uses: wahyd4/kubectl-helm-action@master
+        env:
+          KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
+        with:
+          args: |
+            git clone https://github.com/HXSecurity/DongTai.git
+            helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
+            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}.huoxian.cn \
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.agent_number=iast${{github.run_number}} --set develop.agentZip=${{ env.helm_ns }} --values https://charts.dongtai.io/devops.yaml
+            helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }}-max ./DongTai/deploy/kubernetes/helm/ \
+            --set max=true --set sca.sca_token=${{ secrets.MAX_TOKEN_SCA }} --set usb.usb_token=${{ secrets.MAX_TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}-max.huoxian.cn \
+            --set tag=max-${{ steps.version.outputs.GITHUB_REF }}-latest --set develop.agentZip=${{ env.helm_ns }} \
+            --set build.agent_number=iast${{github.run_number}} --values https://charts.dongtai.io/devops.yaml
diff --git a/.gitignore b/.gitignore
@@ -32,6 +32,8 @@ release
 # for jenv
 .java-version
 
+jmh-result.json
+
 dongtai-api.jar
 dongtai-core.jar
 dongtai-grpc.jar

diff --git a/...-api/src/main/java/io/dongtai/iast/api/gather/spring/extractor/SpringMVCApiExtractor.java b/...-api/src/main/java/io/dongtai/iast/api/gather/spring/extractor/SpringMVCApiExtractor.java
@@ -7,7 +7,8 @@
 import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;
 
 import java.lang.reflect.Method;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * @author CC11001100
@@ -21,11 +22,26 @@ public class SpringMVCApiExtractor {
      * @param applicationContext
      * @return
      */
-    public static OpenApi run(Object applicationContext) {
+    public static List<OpenApi> run(Object applicationContext) {
         WebApplicationContext webApplicationContext = (WebApplicationContext) applicationContext;
         SpringMVCApiExtractor springApplicationContext = new SpringMVCApiExtractor();
-        RequestMappingHandlerMapping requestMappingHandlerMapping = springApplicationContext.findRequestMappingHandlerMapping(webApplicationContext);
-        return new RequestMappingHandlerMappingConvertor(webApplicationContext, requestMappingHandlerMapping).parse();
+        List<RequestMappingHandlerMapping> requestMappingHandlerMappingList = springApplicationContext.findRequestMappingHandlerMapping(webApplicationContext);
+        if (requestMappingHandlerMappingList == null || requestMappingHandlerMappingList.isEmpty()) {
+            DongTaiLog.debug("spring mvc can not find RequestMappingHandlerMapping beans");
+            return null;
+        }
+        return requestMappingHandlerMappingList
+                .stream()
+                .map(mapping -> {
+                    try {
+                        return new RequestMappingHandlerMappingConvertor(webApplicationContext, mapping).parse();
+                    } catch (Throwable e) {
+                        DongTaiLog.debug("spring mvc RequestMappingHandlerMappingConvertor parse error", e);
+                        return null;
+                    }
+                })
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList());
     }
 
     /**
@@ -34,7 +50,7 @@ public static OpenApi run(Object applicationContext) {
      * @param applicationContext
      * @return
      */
-    private RequestMappingHandlerMapping findRequestMappingHandlerMapping(WebApplicationContext applicationContext) {
+    private List<RequestMappingHandlerMapping> findRequestMappingHandlerMapping(WebApplicationContext applicationContext) {
 
         if (applicationContext == null) {
             return null;
@@ -52,15 +68,23 @@ private RequestMappingHandlerMapping findRequestMappingHandlerMapping(WebApplica
                 Map<String, RequestMappingHandlerMapping> m = (Map<String, RequestMappingHandlerMapping>) invoke;
                 RequestMappingHandlerMapping requestMappingHandlerMapping = m.get("requestMappingHandlerMapping");
                 if (requestMappingHandlerMapping != null) {
-                    return requestMappingHandlerMapping;
+                    return Collections.singletonList(requestMappingHandlerMapping);
                 }
             }
         } catch (Throwable e) {
             DongTaiLog.debug("try use BeanFactoryUtils find RequestMappingHandlerMapping exception", e);
         }
 
         // 没有工具类，就只从自己里面找
-        return applicationContext.getBean(RequestMappingHandlerMapping.class);
+        // 2023-7-11 16:58:23 注意，此处可能会寻找到多个，寻找到多个的时候统统上报
+        // case:
+        //         <dependency>
+        //            <groupId>io.springfox</groupId>
+        //            <artifactId>springfox-swagger2</artifactId>
+        //            <version>2.7.0</version>
+        //        </dependency>
+        // 它会创建一个自己的 springfox.documentation.spring.web.PropertySourcedRequestMappingHandlerMapping 继承了 RequestMappingHandlerMapping
+        return new ArrayList<>(applicationContext.getBeansOfType(RequestMappingHandlerMapping.class).values());
     }
 
 }
diff --git a/dongtai-common/src/main/java/io/dongtai/iast/common/constants/AgentConstant.java b/dongtai-common/src/main/java/io/dongtai/iast/common/constants/AgentConstant.java
@@ -1,7 +1,7 @@
 package io.dongtai.iast.common.constants;
 
 public class AgentConstant {
-    public static final String VERSION_VALUE = "v1.12.0-beta1";
+    public static final String VERSION_VALUE = "v1.12.0";
     public static final String LANGUAGE = "JAVA";
     public static final String THREAD_NAME_PREFIX = "DongTai-IAST-";
     public static final String THREAD_NAME_PREFIX_CORE = "DongTai-IAST-Core-";

diff --git a/dongtai-common/src/test/java/io/dongtai/iast/common/utils/AbstractHttpClientUtilsTest.java b/dongtai-common/src/test/java/io/dongtai/iast/common/utils/AbstractHttpClientUtilsTest.java
@@ -21,7 +21,7 @@ public class AbstractHttpClientUtilsTest {
     private final ByteArrayOutputStream outputStreamCaptor = new ByteArrayOutputStream();
     private final boolean oldEnableColor = DongTaiLog.ENABLE_COLOR;
 
-    private static final String BASE_URL = "https://iast.io";
+    private static final String BASE_URL = "https://poc.iast.huoxian.cn";
 
     private void clear() {
         outputStreamCaptor.reset();

diff --git a/dongtai-core/src/main/java/com/secnium/iast/core/AgentEngine.java b/dongtai-core/src/main/java/com/secnium/iast/core/AgentEngine.java
@@ -16,8 +16,7 @@
 
 import java.lang.dongtai.SpyDispatcherHandler;
 import java.lang.instrument.Instrumentation;
-import java.util.ArrayList;
-import java.util.ListIterator;
+import java.util.*;
 
 /**
  * @author [email protected]
@@ -71,6 +70,7 @@ public static void install(String mode, String propertiesFilePath, Integer agent
             DongTaiLog.info("DongTai Engine is successfully installed to the JVM, and it takes {} s",
                     stopWatch.getTime() / 1000);
             DongTaiLog.info("DongTai Agent Version: {}, DongTai Server: {}", AgentConstant.VERSION_VALUE, cfg.getBaseUrl());
+            inject(inst);
             new ServiceDirReport().send();
         } catch (Throwable e) {
             DongTaiLog.error(ErrorCode.get("ENGINE_INSTALL_FAILED"), e);
@@ -134,4 +134,37 @@ private void destroy() {
         }
     }
 
+
+    private static void redefineJavaBaseModule(Instrumentation instrumentation) {
+        if (doesSupportModules()) {
+            try {
+                Instrumentation.class.getMethod("redefineModule", Class.forName("java.lang.Module"), Set.class, Map.class, Map.class, Set.class, Map.class).invoke(instrumentation, getModule(Object.class), Collections.emptySet(), Collections.emptyMap(), Collections.singletonMap("java.lang", Collections.singleton(getModule(EngineManager.class))), Collections.emptySet(), Collections.emptyMap());
+            } catch (Exception e) {
+                DongTaiLog.error(ErrorCode.REDEFINE_MODULE_FAILED,e);
+            }
+        }
+    }
+
+    public static boolean doesSupportModules() {
+        try {
+            Class.forName("java.lang.Module");
+            return true;
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
+    }
+
+    private static Object getModule(Class<?> clazz) {
+        try {
+            return Class.class.getMethod("getModule", new Class[0]).invoke(clazz, new Object[0]);
+        } catch (Exception e) {
+            throw new IllegalStateException("There was a problem while getting the module of the class", e);
+        }
+    }
+    public static void inject(Instrumentation inst) {
+        if (doesSupportModules()) {
+            redefineJavaBaseModule(inst);
+        }
+    }
+
 }
diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/ClassContext.java b/dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/ClassContext.java
@@ -12,6 +12,7 @@ public class ClassContext {
     private String className;
     private String matchedClassName;
     private Set<String> ancestors;
+    private Set<String> matchedClassSet;
     private String superClassName;
     private String[] interfaces;
     private int modifier;
@@ -98,4 +99,12 @@ public boolean isBootstrapClassLoader() {
     public void setBootstrapClassLoader(boolean isBootstrapClassLoader) {
         this.isBootstrapClassLoader = isBootstrapClassLoader;
     }
+
+    public Set<String> getMatchedClassSet() {
+        return matchedClassSet;
+    }
+
+    public void setMatchedClassSet(Set<String> matchedClassSet) {
+        this.matchedClassSet = matchedClassSet;
+    }
 }
diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/asm/AsmMethods.java b/dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/enhance/asm/AsmMethods.java
@@ -256,6 +256,20 @@ static Method getAsmMethod(final Class<?> clazz,
             boolean.class
     );
 
+    Method SPY$skipCollect = InnerHelper.getAsmMethod(
+            SpyDispatcher.class,
+            "skipCollect",
+            Object.class,
+            Object[].class,
+            Object.class,
+            String.class,
+            String.class,
+            String.class,
+            String.class,
+            String.class,
+            boolean.class
+    );
+
     Method SPY$traceFeignInvoke = InnerHelper.getAsmMethod(
             SpyDispatcher.class,
             "traceFeignInvoke",
@@ -279,6 +293,18 @@ static Method getAsmMethod(final Class<?> clazz,
             String.class
     );
 
+    Method SPY$isSkipCollectDubbo = InnerHelper.getAsmMethod(
+            SpyDispatcher.class,
+            "isSkipCollectDubbo",
+            Object.class
+    );
+
+    Method SPY$isSkipCollectFeign = InnerHelper.getAsmMethod(
+            SpyDispatcher.class,
+            "isSkipCollectFeign",
+            Object.class
+    );
+
     Method SPY$reportService = InnerHelper.getAsmMethod(
             SpyDispatcher.class,
             "reportService",

diff --git a/...ore/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/AbstractAdviceAdapter.java b/...ore/src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/AbstractAdviceAdapter.java
@@ -174,6 +174,32 @@ public void captureMethodState(
         pop();
     }
 
+    public void skipCollect(
+            final int opcode,
+            final PolicyNode policyNode,
+            final boolean captureRet
+    ) {
+        newLocal(ASM_TYPE_OBJECT);
+        if (captureRet && !isThrow(opcode)) {
+            loadReturn(opcode);
+        } else {
+            pushNull();
+        }
+        storeLocal(this.nextLocal - 1);
+        invokeStatic(ASM_TYPE_SPY_HANDLER, SPY_HANDLER$getDispatcher);
+        loadThisOrPushNullIfIsStatic();
+        loadArgArray();
+        loadLocal(this.nextLocal - 1);
+        push(policyNode.toString());
+        push(this.context.getClassName());
+        push(this.context.getMatchedClassName());
+        push(this.name);
+        push(this.signature);
+        push(Modifier.isStatic(this.access));
+        invokeInterface(ASM_TYPE_SPY_DISPATCHER, SPY$skipCollect);
+        pop();
+    }
+
     /**
      * 是否抛出异常返回(通过字节码判断)
      *

diff --git a/.../src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/core/DispatchClassPlugin.java b/.../src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/core/DispatchClassPlugin.java
@@ -29,13 +29,13 @@ public DispatchClassPlugin() {
     public ClassVisitor dispatch(ClassVisitor classVisitor, ClassContext classContext, Policy policy) {
         ancestors = classContext.getAncestors();
         className = classContext.getClassName();
-        String matchedClassName = policy.getMatchedClass(className, ancestors);
+        Set<String> matchedClassNameSet = policy.getMatchedClass(classContext,className, ancestors);
 
-        if (null == matchedClassName) {
+        if (0 == matchedClassNameSet.size()) {
             return classVisitor;
         }
 
-        classContext.setMatchedClassName(matchedClassName);
+        classContext.setMatchedClassSet(matchedClassNameSet);
         return new ClassVisit(classVisitor, classContext, policy);
     }
 
@@ -89,17 +89,22 @@ public MethodVisitor visitMethod(final int access, final String name, final Stri
             methodContext.setDescriptor(descriptor);
             methodContext.setParameters(AsmUtils.buildParameterTypes(descriptor));
 
-            String matchedSignature = AsmUtils.buildSignature(context.getMatchedClassName(), name, descriptor);
-
-            mv = lazyAop(mv, access, name, descriptor, matchedSignature, methodContext);
-            boolean methodIsTransformed = mv instanceof MethodAdviceAdapter;
+            String matchedSignature;
+            boolean methodIsTransformed = false;
+            for (String matchedName : context.getMatchedClassSet()) {
+                context.setMatchedClassName(matchedName);
+                matchedSignature = AsmUtils.buildSignature(matchedName, name, descriptor);
+                mv = lazyAop(mv, access, name, descriptor, matchedSignature, methodContext);
+                methodIsTransformed = mv instanceof MethodAdviceAdapter;
+                if (methodIsTransformed) break;
 
+            }
             if (methodIsTransformed && this.classVersion <= Opcodes.V1_6) {
                 mv = new JSRInlinerAdapter(mv, access, name, descriptor, signature, exceptions);
             }
 
             if (methodIsTransformed) {
-                DongTaiLog.trace("rewrite method {} for listener[class={}]", matchedSignature, context.getClassName());
+                DongTaiLog.trace("rewrite method {} for listener[class={}]", context.getMatchedClassName(), context.getClassName());
             }
 
             return mv;

diff --git a/.../src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/core/adapter/SinkAdapter.java b/.../src/main/java/io/dongtai/iast/core/bytecode/enhance/plugin/core/adapter/SinkAdapter.java
@@ -15,6 +15,9 @@ public void onMethodEnter(MethodAdviceAdapter adapter, MethodVisitor mv, MethodC
             if (!(policyNode instanceof SinkNode)) {
                 continue;
             }
+            if ("ssrf".equals(((SinkNode) policyNode).getVulType())){
+                adapter.skipCollect(-1, policyNode, false);
+            }
 
             enterScope(adapter, policyNode);
 

diff --git a/...ast/core/bytecode/enhance/plugin/framework/dubbo/DubboSyncHandlerInvokeAdviceAdapter.java b/...ast/core/bytecode/enhance/plugin/framework/dubbo/DubboSyncHandlerInvokeAdviceAdapter.java
@@ -61,6 +61,7 @@ public void visitMaxs(int maxStack, int maxLocals) {
     }
 
     private void enterMethod() {
+        skipCollect();
         enterScope();
 
         Label elseLabel = new Label();
@@ -113,4 +114,11 @@ private void traceMethod() {
         invokeInterface(ASM_TYPE_SPY_DISPATCHER, SPY$traceDubboInvoke);
         pop();
     }
+
+    private void skipCollect() {
+        invokeStatic(ASM_TYPE_SPY_HANDLER, SPY_HANDLER$getDispatcher);
+        loadArg(0);
+        invokeInterface(ASM_TYPE_SPY_DISPATCHER,SPY$isSkipCollectDubbo);
+        pop();
+    }
 }
diff --git a/...ast/core/bytecode/enhance/plugin/framework/feign/FeignSyncHandlerInvokeAdviceAdapter.java b/...ast/core/bytecode/enhance/plugin/framework/feign/FeignSyncHandlerInvokeAdviceAdapter.java
@@ -44,6 +44,7 @@ public void visitMaxs(int maxStack, int maxLocals) {
     }
 
     private void enterMethod() {
+        skipCollect();
         enterScope();
 
         Label elseLabel = new Label();
@@ -89,4 +90,11 @@ private void traceMethod() {
         invokeInterface(ASM_TYPE_SPY_DISPATCHER, SPY$traceFeignInvoke);
         pop();
     }
+
+    private void skipCollect() {
+        invokeStatic(ASM_TYPE_SPY_HANDLER, SPY_HANDLER$getDispatcher);
+        loadThisOrPushNullIfIsStatic();
+        invokeInterface(ASM_TYPE_SPY_DISPATCHER,SPY$isSkipCollectFeign);
+        pop();
+    }
 }