From 52b6fe6f0beaf0842bfb8f49c8301684d20424bb Mon Sep 17 00:00:00 2001 From: lorryyzwu Date: Tue, 21 Jan 2025 16:23:34 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E6=8F=92=E5=85=A5MyS?= =?UTF-8?q?QL=E6=97=B6=E8=84=9A=E6=9C=AC=E5=8F=82=E6=95=B0=E8=BF=87?= =?UTF-8?q?=E9=95=BF=E7=9A=84=E6=8A=A5=E9=94=99=E4=BF=A1=E6=81=AF=EF=BC=8C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8validation=E6=A0=A1=E9=AA=8C=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E9=95=BF=E5=BA=A6=20#3374?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bk/job/common/util/Base64Util.java | 24 ++++++ .../bk/job/common/util/Base64UtilTest.java | 81 +++++++++++++++++++ ...QLDataType.java => MySQLTextDataType.java} | 16 ++-- ...ava => NotExceedMySQLTextFieldLength.java} | 45 +++++++---- .../request/EsbFastExecuteScriptRequest.java | 14 ++-- ...EsbBkCIPluginFastExecuteScriptRequest.java | 14 +++- .../EsbFastExecuteScriptV3Request.java | 14 ++-- .../request/WebFastExecuteScriptRequest.java | 14 ++-- 8 files changed, 177 insertions(+), 45 deletions(-) create mode 100644 src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/Base64UtilTest.java rename src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/{MySQLDataType.java => MySQLTextDataType.java} (76%) rename src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/{NotExceedMySQLFieldLength.java => NotExceedMySQLTextFieldLength.java} (63%) diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java index 1968d4c5c8..c532d81bb9 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java @@ -103,4 +103,28 @@ public static byte[] decodeContentToByte(String content) { throw e; } } + + /** + * 根据BASE64编码后的字符串计算原始字节流长度，不用decode字符串 + * + * @param encodedContent 编码后的字符串 + * @return 原始字符串的长度 + */ + public static int calOriginBytesLength(String encodedContent) { + if (StringUtils.isEmpty(encodedContent)) { + return 0; + } + + // 最多只会填充两个 = + int fillCount = 0; + for (int i = 0; i <= 1 && encodedContent.length() - 1 - i >= 0; i++) { + if (encodedContent.charAt(encodedContent.length() - 1 - i) == '=') { + fillCount++; + } else { + break; + } + } + + return encodedContent.length() * 3 / 4 - fillCount; + } } diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/Base64UtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/Base64UtilTest.java new file mode 100644 index 0000000000..c2ad9286c9 --- /dev/null +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/Base64UtilTest.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +import org.junit.jupiter.api.Test; + +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class Base64UtilTest { + + @Test + void testCalOriginBytesLengthByEncodedContent() { + for (int i = 0; i < 100; i++) { + String originStr = genRandomString(); + assertCal(originStr); + } + String specialStrWithChinese = "# 在当前脚本执行时，第一行输出当前时间和进程ID，详见上面函数：job_get_now\n" + + "job_start\n" + + "\n" + + "###### 作业平台中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值\n" + + "###### 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败\n" + + "###### 可在此处开始编写您的脚本逻辑代码"; + assertCal(specialStrWithChinese); + + } + + private void assertCal(String originStr) { + + String encodedStr = Base64Util.encodeContentToStr(originStr); + int calLengthByUtil = Base64Util.calOriginBytesLength(encodedStr); + + byte[] bytes = Base64Util.decodeContentToByte(encodedStr); + int originBytesLength = bytes.length; + + System.out.println("originStr:" + originStr); + System.out.println("originBytesLength:" + originBytesLength); + System.out.println("calLengthByUtil:" + calLengthByUtil); + + assertEquals(originBytesLength, calLengthByUtil, + "bytes length should be " + originBytesLength + ", but is " + calLengthByUtil); + } + + private String genRandomString() { + final int ASCII_LOW = 32; + final int ASCII_HIGH = 126; + final int SHORTEST = 1; + final int LONGEST = 500; + Random rand = new Random(); + int length = rand.nextInt(LONGEST - SHORTEST) + SHORTEST; + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + int randomAscii = ASCII_LOW + rand.nextInt(ASCII_HIGH - ASCII_LOW + 1); + sb.append((char) randomAscii); + } + return sb.toString(); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLDataType.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLTextDataType.java similarity index 76% rename from src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLDataType.java rename to src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLTextDataType.java index 9605bd27ca..f382131185 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLDataType.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLTextDataType.java @@ -27,27 +27,27 @@ import lombok.Getter; /** - * 这里暂时只记录字符串形式的MySQL类型，用于获取各类型的最大长度 + * 这里暂时只记录TEXT形式的MySQL类型，用于获取各类型的最大长度。 + * 因为在MySQL中，TEXT类型的存储是以字节流的长度为限制的，而char/varchar则是以字符串的长度为限制的，所以校验长度合法的逻辑是不太一样的。 + * */ @Getter -public enum MySQLDataType { - CHAR("CHAR", 255L), - VARCHAR("VARCHAR", 65535L), +public enum MySQLTextDataType { TINYTEXT("TINYTEXT", 255L), TEXT("TEXT", 65535L), MEDIUMTEXT("MEDIUMTEXT", 16777215L), LONGTEXT("LONGTEXT", 4294967295L); private final String value; - private final Long maximumLength; + private final Long maximumLength; // MySQL能存的最大字节数 - MySQLDataType(String value, Long maximumLength) { + MySQLTextDataType(String value, Long maximumLength) { this.value = value; this.maximumLength = maximumLength; } - public static MySQLDataType valOf(String value) { - for (MySQLDataType type : MySQLDataType.values()) { + public static MySQLTextDataType valOf(String value) { + for (MySQLTextDataType type : MySQLTextDataType.values()) { if (type.value.equals(value)) { return type; } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLFieldLength.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLTextFieldLength.java similarity index 63% rename from src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLFieldLength.java rename to src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLTextFieldLength.java index 81d1a00191..ab4e49762a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLFieldLength.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLTextFieldLength.java @@ -25,7 +25,8 @@ package com.tencent.bk.job.common.validation; -import com.tencent.bk.job.common.constant.MySQLDataType; +import com.tencent.bk.job.common.constant.MySQLTextDataType; +import com.tencent.bk.job.common.util.Base64Util; import lombok.extern.slf4j.Slf4j; import javax.validation.Constraint; @@ -35,6 +36,7 @@ import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.Target; +import java.nio.charset.StandardCharsets; import static java.lang.annotation.ElementType.ANNOTATION_TYPE; import static java.lang.annotation.ElementType.FIELD; @@ -44,13 +46,16 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; /** - * 通过在MySQL定义的字段类型判断是否过长 + * 通过在MySQL定义的TEXT一族字段类型（TINYTEXT/TEXT/MEDIUMTEXT/LONGTEXT）判断是否过长 + * 因为在MySQL中，TEXT类型的存储是以字节流的长度为限制的，而char/varchar则是以字符串的长度为限制的，所以校验长度合法的逻辑是不太一样的 + * 因此这个校验器只校验TEXT类型的字节流长度，若需要校验char/varchar类型的字段长度，请用str.length()校验 + * */ @Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) -@Constraint(validatedBy = NotExceedMySQLFieldLength.FieldLengthValidator.class) +@Constraint(validatedBy = NotExceedMySQLTextFieldLength.FieldLengthValidator.class) @Documented @Retention(RUNTIME) -public @interface NotExceedMySQLFieldLength { +public @interface NotExceedMySQLTextFieldLength { String message() default "{fieldName} {validation.constraints.NotExceedMySQLFieldLength.message}"; @@ -60,29 +65,39 @@ String fieldName(); - MySQLDataType fieldType(); + MySQLTextDataType fieldType(); + + boolean base64(); @Slf4j - class FieldLengthValidator implements ConstraintValidator { + class FieldLengthValidator implements ConstraintValidator { - MySQLDataType fieldType = null; + MySQLTextDataType fieldType = null; + boolean useBase64 = false; @Override - public void initialize(NotExceedMySQLFieldLength constraintAnnotation) { + public void initialize(NotExceedMySQLTextFieldLength constraintAnnotation) { fieldType = constraintAnnotation.fieldType(); + useBase64 = constraintAnnotation.base64(); } @Override - public boolean isValid(String scriptContent, ConstraintValidatorContext constraintValidatorContext) { - - log.debug("[Validate MySQLFieldLength] field type: {}, content length: {}, maximum length: {}]", - fieldType.getValue(), scriptContent.length(), fieldType.getMaximumLength()); - - if (fieldType == null || scriptContent == null) { + public boolean isValid(String value, ConstraintValidatorContext constraintValidatorContext) { + if (value == null || fieldType == null) { return true; } - return scriptContent.length() <= fieldType.getMaximumLength(); + int currentLength; + if (useBase64) { + currentLength = Base64Util.calOriginBytesLength(value); + } else { + currentLength = value.getBytes(StandardCharsets.UTF_8).length; + } + + log.debug("[Validate MySQLFieldLength] field type: {}, current length: {}, maximum length: {}]", + fieldType.getValue(), currentLength, fieldType.getMaximumLength()); + + return currentLength <= fieldType.getMaximumLength(); } } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java index ce3673c2a5..c06c7e74b8 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java @@ -26,11 +26,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.constant.MySQLDataType; +import com.tencent.bk.job.common.constant.MySQLTextDataType; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.EsbServerDTO; -import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import lombok.Getter; import lombok.Setter; import org.hibernate.validator.constraints.Range; @@ -54,9 +54,10 @@ public class EsbFastExecuteScriptRequest extends EsbAppScopeReq { * "脚本内容，BASE64编码 */ @JsonProperty("script_content") - @NotExceedMySQLFieldLength( + @NotExceedMySQLTextFieldLength( fieldName = "scriptContent", - fieldType = MySQLDataType.MEDIUMTEXT + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true ) private String content; @@ -75,9 +76,10 @@ public class EsbFastExecuteScriptRequest extends EsbAppScopeReq { * 脚本参数， BASE64编码 */ @JsonProperty("script_param") - @NotExceedMySQLFieldLength( + @NotExceedMySQLTextFieldLength( fieldName = "scriptParam", - fieldType = MySQLDataType.TEXT + fieldType = MySQLTextDataType.TEXT, + base64 = true ) private String scriptParam; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastExecuteScriptRequest.java index 4dff69b62d..b9a91865d4 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastExecuteScriptRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastExecuteScriptRequest.java @@ -26,11 +26,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.constant.MySQLDataType; +import com.tencent.bk.job.common.constant.MySQLTextDataType; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteTargetDTO; import com.tencent.bk.job.common.validation.CheckEnum; -import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import com.tencent.bk.job.common.validation.ValidationGroups; import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.validator.EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider; @@ -63,9 +63,10 @@ public class EsbBkCIPluginFastExecuteScriptRequest extends EsbAppScopeReq { * 脚本内容，BASE64编码 */ @JsonProperty("script_content") - @NotExceedMySQLFieldLength( + @NotExceedMySQLTextFieldLength( fieldName = "scriptContent", - fieldType = MySQLDataType.MEDIUMTEXT + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true ) private String content; @@ -97,6 +98,11 @@ public class EsbBkCIPluginFastExecuteScriptRequest extends EsbAppScopeReq { * 脚本参数， BASE64编码 */ @JsonProperty("script_param") + @NotExceedMySQLTextFieldLength( + fieldName = "scriptParam", + fieldType = MySQLTextDataType.TEXT, + base64 = true + ) private String scriptParam; /** diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java index 3b4c2b04a3..d7a675f3a2 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java @@ -26,11 +26,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.constant.MySQLDataType; +import com.tencent.bk.job.common.constant.MySQLTextDataType; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; -import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; import lombok.Getter; import lombok.Setter; @@ -56,9 +56,10 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq { * "脚本内容，BASE64编码 */ @JsonProperty("script_content") - @NotExceedMySQLFieldLength( + @NotExceedMySQLTextFieldLength( fieldName = "scriptContent", - fieldType = MySQLDataType.MEDIUMTEXT + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true ) private String content; @@ -84,9 +85,10 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq { * 脚本参数， BASE64编码 */ @JsonProperty("script_param") - @NotExceedMySQLFieldLength( + @NotExceedMySQLTextFieldLength( fieldName = "scriptParam", - fieldType = MySQLDataType.TEXT + fieldType = MySQLTextDataType.TEXT, + base64 = true ) private String scriptParam; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java index 266a6e83b3..b0b5e20891 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java @@ -28,9 +28,9 @@ import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.constant.MySQLDataType; +import com.tencent.bk.job.common.constant.MySQLTextDataType; import com.tencent.bk.job.common.model.vo.TaskTargetVO; -import com.tencent.bk.job.common.validation.NotExceedMySQLFieldLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -54,9 +54,10 @@ public class WebFastExecuteScriptRequest { * 脚本内容 */ @ApiModelProperty(value = "脚本内容，BASE64编码，当手动录入的时候使用此参数") - @NotExceedMySQLFieldLength( + @NotExceedMySQLTextFieldLength( fieldName = "scriptContent", - fieldType = MySQLDataType.MEDIUMTEXT + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true ) private String content; @@ -88,9 +89,10 @@ public class WebFastExecuteScriptRequest { * 脚本参数 */ @ApiModelProperty(value = "脚本参数") - @NotExceedMySQLFieldLength( + @NotExceedMySQLTextFieldLength( fieldName = "scriptParam", - fieldType = MySQLDataType.TEXT + fieldType = MySQLTextDataType.TEXT, + base64 = true ) private String scriptParam;