diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80e02dd --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +.idea/ +.gradle/ +build/ +target/ +out/ +release/ +reports/ +*.iml +.DS_Store +*.log +gradle/ +gradlew +gradlew.bat +version.txt +build.yml +.codecc +task.json diff --git a/README.md b/README.md index 8c5f1b0..7462671 100644 --- a/README.md +++ b/README.md @@ -1 +1,57 @@ -# run \ No newline at end of file +#### 插件功能 +在构建机上执行脚本： +- 默认情况使用Bash进行脚本执行 +- 可选解析器如图所示![command.png](./img/command.png) + +#### 适用场景 +执行编译脚本 + +支持通过如下方式设置当前步骤输出变量： +``` +echo "::set-output name=::" +``` +如： +``` +echo "::set-output name=release_type::dev" +``` +在下游步骤入参中: +- 通过 ${{ jobs..steps..outputs.release_type }} 引用此变量值 +- 为当前Job上配置的 Job ID +- 为当前 Task 上配置的 Step ID + +支持通过如下方式设置/修改流水线变量： +``` +echo "::set-variable name=::" +``` +如： +``` +echo "::set-variable name=a::1" +``` +在下游步骤入参中，通过 ${{ variables.a }} 方式引用此变量 + +#### 使用限制和受限解决方案[可选] +设置输出参数或者流水线变量时，**在当前步骤不会生效，在下游步骤才生效** + +#### 常见的失败原因和解决方案 +1. 脚本执行退出码非0时，当前步骤执行结果为失败，请检查脚本逻辑，或确认执行环境是否满足需求 + +#### 多行文本使用set-output/set-variable/set-gate-value +bash示例： +使用format_multiple_lines 替代echo输出 +``` +content=$(ls -l ..) +echo "$content" +format_multiple_lines "::set-output name=content_a::$content" +resultStr="\n" +resultStr="$resultStr\n$PATH" +resultStr="$resultStr\n$PATH" +resultStr="$resultStr\n$PATH" +echo resultStr=$resultStr +format_multiple_lines "::set-output name=content2_a::$resultStr" +``` + +python 示例： +``` +multiple_lines = "line one \n line two \n line three" +print("::set-output name=lines::{0}".format(format_multiple_lines(multiple_lines))) +``` \ No newline at end of file diff --git a/img/command.png b/img/command.png new file mode 100644 index 0000000..78c2179 Binary files /dev/null and b/img/command.png differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..04f43e0 --- /dev/null +++ b/pom.xml @@ -0,0 +1,107 @@ + + + 4.0.0 + + com.tencent.devops.ci-plugins + run + 1.0.0 + + + + 1.1.3 + 1.8 + 1.7.0 + ${java.version} + ${java.version} + + UTF-8 + + + + + + com.tencent.devops.ci-plugins + java-plugin-sdk + ${sdk.version} + + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin.version} + + + junit + junit + 4.12 + + + org.apache.commons + commons-exec + 1.3 + + + + + ${project.name} + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + package + + single + + + + jar-with-dependencies + + + + com.tencent.bk.devops.atom.AtomRunner + + + + + + + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + compile + + compile + + + + ${project.basedir}/src/main/kotlin + ${project.basedir}/src/main/java + + + + + test-compile + + test-compile + + + + ${project.basedir}/src/test/kotlin + ${project.basedir}/src/test/java + + + + + + + + + diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/ScriptRunAtom.kt b/src/main/kotlin/com/tencent/bk/devops/atom/ScriptRunAtom.kt new file mode 100644 index 0000000..ad19abb --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/ScriptRunAtom.kt @@ -0,0 +1,538 @@ +package com.tencent.bk.devops.atom + +import com.tencent.bk.devops.atom.api.QualityApi +import com.tencent.bk.devops.atom.common.CI_TOKEN_CONTEXT +import com.tencent.bk.devops.atom.common.ErrorCode +import com.tencent.bk.devops.atom.common.JOB_OS_CONTEXT +import com.tencent.bk.devops.atom.common.Status +import com.tencent.bk.devops.atom.common.WORKSPACE_CONTEXT +import com.tencent.bk.devops.atom.common.utils.ReplacementUtils +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.enums.OSType +import com.tencent.bk.devops.atom.exception.AtomException +import com.tencent.bk.devops.atom.pojo.AdditionalOptions +import com.tencent.bk.devops.atom.pojo.AgentEnv.getOS +import com.tencent.bk.devops.atom.pojo.ArtifactData +import com.tencent.bk.devops.atom.pojo.AtomResult +import com.tencent.bk.devops.atom.pojo.ReportData +import com.tencent.bk.devops.atom.pojo.ScriptRunAtomParam +import com.tencent.bk.devops.atom.pojo.ShellType +import com.tencent.bk.devops.atom.pojo.StringData +import com.tencent.bk.devops.atom.pojo.request.IndicatorCreate +import com.tencent.bk.devops.atom.pojo.request.QualityDataType +import com.tencent.bk.devops.atom.pojo.request.QualityOperation +import com.tencent.bk.devops.atom.spi.AtomService +import com.tencent.bk.devops.atom.spi.TaskAtom +import com.tencent.bk.devops.atom.utils.CommandLineUtils +import com.tencent.bk.devops.atom.utils.ScriptEnvUtils +import com.tencent.bk.devops.atom.utils.script.BashUtil +import com.tencent.bk.devops.atom.utils.script.BatScriptUtil +import com.tencent.bk.devops.atom.utils.script.PowerShellUtil +import com.tencent.bk.devops.atom.utils.script.PwshUtil +import com.tencent.bk.devops.atom.utils.script.PythonUtil +import com.tencent.bk.devops.atom.utils.script.ShUtil +import com.tencent.bk.devops.plugin.pojo.ErrorType +import org.apache.commons.lang3.StringUtils +import org.slf4j.LoggerFactory +import java.io.File + +/** + * @version 1.0.0 + */ +@AtomService(paramClass = ScriptRunAtomParam::class) +class ScriptRunAtom : TaskAtom { + + private val qualityApi = QualityApi() + + private val QUALITY_BOOLEAN_OPERATIONS = listOf(QualityOperation.EQ) + private val QUALITY_ALL_OPERATIONS = listOf( + QualityOperation.EQ, + QualityOperation.GT, + QualityOperation.GE, + QualityOperation.LT, + QualityOperation.LE + ) + + private val USER_ERROR_MESSAGE = """ +======脚本执行失败,问题排查指引====== +当脚本退出码非0时，执行失败。可以从以下路径进行分析： +1. 根据错误日志排查 +2. 在本地手动执行脚本。如果本地执行也失败，很可能是脚本逻辑问题； +如果本地OK，排查构建环境（比如环境依赖、或者代码变更等） + """ + + override fun execute(atomContext: AtomContext) { + + val param = atomContext.param as ScriptRunAtomParam + val result = atomContext.result + // 检查参数 + checkParam(param, result) + if (result.status != Status.success) { + return + } + val script = param.script + // 字符集选择，应对某些windows构建存在的字符集不匹配的问题 + val charSetType: CharsetType = try { + CharsetType.valueOf(param.charSetType) + } catch (ignore: Throwable) { + CharsetType.DEFAULT + } + // 获取运行时变量 + val runtimeVariables = atomContext.allParameters.map { it.key to it.value.toString() }.toMap() + // 获取系统类型 + val osType = getOS() + // 获取构建id + val buildId = atomContext.param.pipelineBuildId + /*拿到工作目录，后续文件操作将在工作目录进行*/ + val workspace = File(param.bkWorkspace) + /*替换脚本变量，目前变量已在引擎替换，此次预留暂时没有起实际作用*/ + val realCommand = parseTemplate(script, emptyMap(), workspace) + /*获取input中的变量，为了后续塞环境变量时不会意外将其塞入*/ + val paramClassName = param.javaClass.declaredFields.map { it.name }.toList() + + handle(result) { + /*获取脚本类型和系统类型*/ + val additionalOptions = AdditionalOptions(param.shell) + /*检查脚本类型是否适配当前系统*/ + checkOS(additionalOptions.shell, result) + try { + when (additionalOptions.shell) { + /*batch脚本，windows使用*/ + ShellType.CMD -> BatScriptUtil.execute( + script = realCommand, + buildId = buildId, + runtimeVariables = runtimeVariables, + dir = workspace, + charsetType = charSetType, + paramClassName = paramClassName + ) + /*shell脚本，一般linux和macos使用*/ + ShellType.BASH -> BashUtil.execute( + script = realCommand, + buildId = buildId, + runtimeVariables = runtimeVariables, + dir = workspace, + // 市场插件执行时buildEnvs已经写在环境变量中，作为子进程可以直接读取 + buildEnvs = emptyList(), + stepId = param.stepId, + paramClassName = paramClassName + ) + /*python脚本，需要目标构建机安装python3环境*/ + ShellType.PYTHON -> PythonUtil.execute( + script = realCommand, + buildId = buildId, + runtimeVariables = runtimeVariables, + dir = workspace, + buildEnvs = emptyList(), + stepId = param.stepId, + charsetType = charSetType, + paramClassName = paramClassName + ) + /*powershell脚本*/ + ShellType.POWERSHELL_CORE -> PwshUtil.execute( + script = realCommand, + buildId = buildId, + runtimeVariables = runtimeVariables, + dir = workspace, + stepId = param.stepId, + paramClassName = paramClassName + ) + /*powershell desktop脚本*/ + ShellType.POWERSHELL_DESKTOP -> PowerShellUtil.execute( + script = realCommand, + buildId = buildId, + runtimeVariables = runtimeVariables, + dir = workspace, + stepId = param.stepId, + paramClassName = paramClassName + ) + /*执行sh命令的脚本*/ + ShellType.SH -> ShUtil.execute( + script = realCommand, + buildId = buildId, + runtimeVariables = runtimeVariables, + dir = workspace, + // 市场插件执行时buildEnvs已经写在环境变量中，作为子进程可以直接读取 + buildEnvs = emptyList(), + stepId = param.stepId, + paramClassName = paramClassName + ) + else -> {} + } + + result.status = Status.success + result.message = "$osType 脚本执行成功" + } catch (taskError: AtomException) { + /*处理普通异常，这里是脚本逻辑抛出的异常*/ + logger.warn("Fail to run the script task") + logger.debug("TaskExecuteException|${taskError.message}", taskError) + result.status = Status.failure + result.message = "$osType 脚本执行失败" + /*返回失败以及对应的异常类型*/ + throw AtomException( + taskError.message + "\n$USER_ERROR_MESSAGE" + ) + } catch (ignore: Throwable) { + /*处理意外发生的异常，全局捕获*/ + logger.warn("Fail to run the script task") + logger.debug("Throwable|${ignore.message}", ignore) + result.status = Status.failure + result.message = "$osType 脚本执行失败" + /*返回user类型错误，一般用户使用错误会引起这种情况*/ + throw AtomException( + USER_ERROR_MESSAGE + ) + } finally { + // 写入上下文 + ScriptEnvUtils.getContext(buildId, workspace).plus(parseContextFromMultiLine(buildId, workspace)) + .forEach { (key, value) -> + /*根据拆分的格式来区分具体的类型*/ + val split = key.split(",") + when (split.size) { + /*只有一位，只需要输出为上下文变量格式*/ + 1 -> atomContext.result.data[key] = StringData(value) + /*有5位时区分了3种类型情况*/ + 5 -> + // 以逗号为分隔符 左右依次为name type label path reportType + atomContext.result.data[split[0]] = when (split[1]) { + /*指定string的类型输出为上下文*/ + "string" -> StringData(value) + /*指定位artifact的类型输出为构件*/ + "artifact" -> ArtifactData(setOf(value)) + /*指定为report 输出为报告*/ + "report" -> { + /*第三方报告*/ + if (split[4].contains("THIRDPARTY")) { + ReportData(split[2], value) + /*本地报告*/ + } else if (split[3].isNotBlank()) { + ReportData(split[2], split[3], value) + } else { + /*不支持其他report类型，如果有直接抛错*/ + throw AtomException( + "脚本执行失败 set-output report设置有误，请检查" + ) + } + } + /*不支持其他set-output类型，如果有直接抛错*/ + else -> throw AtomException( + "脚本执行失败 set-output设置有误，请检查: $split" + ) + } + /*不支持其他类型，如果有直接抛错*/ + else -> throw AtomException( + "脚本执行失败 set-output或set-variable设置有误，请检查: [${split.size == 1}]$split" + ) + } + } + // 写入环境变量 + ScriptEnvUtils.getEnv(buildId, workspace).forEach { (key, value) -> + atomContext.result.data[key] = StringData(value) + } + // 写入质量红线 + setGatewayValue(atomContext, workspace) + + /*脚本执行结束清理临时文件*/ + ScriptEnvUtils.cleanWhenEnd(buildId, workspace) + } + } + } + + /** + * 单独解析多行内容的输出 + */ + private fun parseContextFromMultiLine(buildId: String, workspace: File): Map { + val res = mutableMapOf() + /*获得多行的输出内容*/ + ScriptEnvUtils.getMultipleLines(buildId, workspace).forEach { + logger.debug("multiLine:$it") + /*解析variable或者output变量*/ + val str = CommandLineUtils.parseVariable(it) ?: CommandLineUtils.parseOutput(it) + if (str != null) { + val split = str.split("=", ignoreCase = false, limit = 2) + /*解码替换回来，得到多行内容*/ + res[split[0].trim()] = escapeData(split[1].trim()) + } + } + return res + } + + /** + * 统一的异常处理模块 + */ + private fun handle( + atomResult: AtomResult, + action: () -> T? + ) { + try { + action() + } catch (triggerE: AtomException) { + atomResult.message = triggerE.message + atomResult.errorCode = ErrorCode.USER_SCRIPT_TASK_FAIL + atomResult.errorType = ErrorType.USER.num + atomResult.status = Status.failure + } catch (e: Throwable) { + // unknown 情况归属为插件问题，需要插件方来处理 + atomResult.message = "Unknown Error: " + e.message + atomResult.errorCode = ErrorCode.USER_TASK_OPERATE_FAIL + atomResult.errorType = ErrorType.PLUGIN.num + atomResult.status = Status.error + } + } + + private fun escapeData(value: String): String { + return value + /*单引号的情况，是batch替换的结果，这里进行兼容*/ + .replace("'%0D'", "\r") + .replace("%0D", "\r") + .replace("'%0A'", "\n") + .replace("%0A", "\n") + .replace("'%25'", "%") + .replace("%25", "%") + } + + /** + * 设置红线指标 + */ + private fun setGatewayValue(atomContext: AtomContext, workspace: File) { + /*去拿红线指标输出文件*/ + val gatewayFile = File(workspace, ScriptEnvUtils.getQualityGatewayEnvFile()) + try { + /*不存在直接推出*/ + if (!gatewayFile.exists()) return + val data = mutableMapOf() + val title = mutableMapOf() + // 创建红线指标 + gatewayFile.readLines().forEach { + val split = it.split(",") + if (split.size > 2) { + /*格式不对直接抛错*/ + throw AtomException( + "much gateway parameter,count:${split.size}" + ) + } + val nameToValue = split.getOrNull(0) + val nameToTitle = split.getOrNull(1) + /*去做二次切分*/ + keyEqualValueInsertMap(nameToValue, data) + keyEqualValueInsertMap(nameToTitle, title) + } + /*创建红线指标*/ + updateIndicatorTitle( + userId = atomContext.param.pipelineStartUserId, + projectId = atomContext.param.projectName, + data = data, + nameMapToTitle = title + ) + // 将自定义指标的值入库 + saveQualityData( + taskId = atomContext.param.pipelineTaskId, + taskName = atomContext.param.taskName, + data = data + ) + + logger.info("save gateway value: $data") + } catch (ignore: Exception) { + /*处理异常*/ + logger.info("save gateway value fail: ${ignore.message}") + logger.error("setGatewayValue|${ignore.message}", ignore) + } finally { + /*执行完后删除文件*/ + gatewayFile.delete() + } + } + + private fun keyEqualValueInsertMap(nameToValue: String?, map: MutableMap) { + /*为空直接退出*/ + if (nameToValue.isNullOrBlank()) return + /*二次切分后确定最终的 key*/ + val key = nameToValue.split("=").getOrNull(0) ?: throw AtomException( + "Illegal gateway key set: $nameToValue" + ) + /*二次切分后确定最终的 value*/ + val value = nameToValue.split("=").getOrNull(1) ?: throw AtomException( + "Illegal gateway key set: $nameToValue" + ) + /*组装进map*/ + map[key] = value.trim() + } + +/* + private fun upsertIndicator( + userId: String, + projectId: String, + data: Map + ) { + val indicatorCreates = data.map { (name, value) -> + val dataType = getQualityDataType(value) + IndicatorCreate( + name = name, + cnName = name, + desc = "", + dataType = dataType, + operation = getQualityOperations(dataType == QualityDataType.BOOLEAN), + threshold = value, + elementType = QUALITY_ELEMENT_TYPE + ) + } + doUpsertIndicator(userId = userId, projectId = projectId, indicatorCreates = indicatorCreates) + }*/ + + private fun updateIndicatorTitle( + userId: String, + projectId: String, + data: Map, + nameMapToTitle: Map + ) { + // dataMapExample: pass_rate to 1.0 + val indicatorCreates = data.map { (name, value) -> + /*获取数据类型*/ + val dataType = getQualityDataType(value) + /*获取标题*/ + val title = getTitleOrDefault(name, nameMapToTitle) + IndicatorCreate( + name = name, + cnName = title, + dataType = dataType + ) + } + /*调接口创建红线指标*/ + doUpsertIndicator(userId = userId, projectId = projectId, indicatorCreates = indicatorCreates) + } + + /*获取标题的逻辑*/ + private fun getTitleOrDefault( + name: String, + nameMapToTitle: Map + ): String { + val title = nameMapToTitle[name] + /*变量中有则返回变量中的name*/ + if (title.isNullOrBlank()) { + return name + } + return title + } + + private fun doUpsertIndicator( + userId: String, + projectId: String, + indicatorCreates: List + ) { + /*通过接口创建红线指标*/ + qualityApi.upsertIndicator( + userId = userId, projectId = projectId, indicatorCreate = indicatorCreates + ).data.let { + if (it == null || !it) { + /*返回异常情况处理*/ + throw AtomException( + "创建 run 红线指标失败" + ) + } + } + } + + fun getQualityDataType(value: String): QualityDataType { + /*转int*/ + value.toIntOrNull().let { + if (it != null) { + return QualityDataType.INT + } + } + /*转float*/ + value.toFloatOrNull().let { + if (it != null) { + return QualityDataType.FLOAT + } + } + /*转boolean*/ + if (value == "true" || value == "false") { + return QualityDataType.BOOLEAN + } + /*其他类型直接抛错*/ + throw AtomException( + "gateWay error qualityDataType: $value,only support INT、FLOAT、BOOLEAN" + ) + } + +/* + private fun getQualityOperations(isBoolean: Boolean) = if (isBoolean) { + QUALITY_BOOLEAN_OPERATIONS + } else { + QUALITY_ALL_OPERATIONS + }*/ + + private fun saveQualityData( + taskId: String, + taskName: String, + data: Map + ) { + /*调接口保存红线数据*/ + qualityApi.saveScriptHisMetadata(taskId = taskId, taskName = taskName, data = data).data.let { + if (it == null || !it) { + /*返回异常处理*/ + throw AtomException( + "保存 run 红线数据失败" + ) + } + } + } + + /** + * 检查参数 + * @param param 请求参数 + * @param result 结果 + */ + private fun checkParam(param: ScriptRunAtomParam, result: AtomResult) { + // 参数检查 + if (StringUtils.isBlank(param.script)) { + result.status = Status.failure // 状态设置为失败 + result.message = "脚本内容不能为空" // 失败信息回传给插件执行框架会打印出结果 + } + } + + private fun parseTemplate(command: String, data: Map, dir: File): String { + /*通用变量替换逻辑*/ + return ReplacementUtils.replace( + command = command, + replacement = object : ReplacementUtils.KeyReplacement { + override fun getReplacement(key: String): String? = if (data[key] != null) { + data[key] + } else { + null + } + }, + /*额外需要替换的变量*/ + contextMap = mapOf( + WORKSPACE_CONTEXT to dir.absolutePath, + CI_TOKEN_CONTEXT to (data[CI_TOKEN_CONTEXT] ?: ""), + JOB_OS_CONTEXT to getOS().name + ) + ) + } + + /* + * 检查系统类型 + */ + private fun checkOS(shellType: ShellType, result: AtomResult) { + val os = getOS() + when { + /*非windows不能使用cmd（batch）*/ + os != OSType.WINDOWS && shellType == ShellType.CMD || + /*非windows不能使用POWERSHELL_DESKTOP*/ + os != OSType.WINDOWS && shellType == ShellType.POWERSHELL_DESKTOP || + /*windows不能使用sh*/ + os == OSType.WINDOWS && shellType == ShellType.SH -> { + result.status = Status.failure + result.message = "$os 脚本执行失败" + /*不满足的情况直接用户抛错*/ + throw AtomException( + "The current system(${os.name}) does not support: ${shellType.shellName}" + ) + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(ScriptRunAtom::class.java) + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/ScriptRunAtomTest.kt b/src/main/kotlin/com/tencent/bk/devops/atom/ScriptRunAtomTest.kt new file mode 100644 index 0000000..2564894 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/ScriptRunAtomTest.kt @@ -0,0 +1,232 @@ +package com.tencent.bk.devops.atom + +import com.tencent.bk.devops.atom.pojo.ScriptRunAtomParam +import junit.framework.TestCase + +class ScriptRunAtomTest : TestCase() { + + private val atom = ScriptRunAtom() + + fun testRun() { + atom.execute(AtomContext(ScriptRunAtomParam::class.java)) + } +/* + fun testGetQualityDataType() { + val cls = ScriptRunAtom() + assertEquals(QualityDataType.INT, cls.getQualityDataType("0")) + assertEquals(QualityDataType.FLOAT, cls.getQualityDataType("0.123")) + assertEquals(QualityDataType.BOOLEAN, cls.getQualityDataType("true")) + try { + cls.getQualityDataType("xxx") + } catch (e: TaskExecuteException) { + assertEquals(e.errorMsg, "error qualityDataType: xxx") + } catch (e: Exception) { + assert(false) + } + } + + companion object { + private val OUTPUT_NAME = Pattern.compile("name=([^,:=\\s]*)") + private val OUTPUT_TYPE = Pattern.compile("type=([^,:=\\s]*)") + private val OUTPUT_LABEL = Pattern.compile("label=([^,:=\\s]*)") + private val OUTPUT_PATH = Pattern.compile("path=([^,:=\\s]*)") + private val OUTPUT_REPORT_TYPE = Pattern.compile("reportType=([^,:=\\s]*)") + private val OUTPUT_GATE_TITLE = Pattern.compile("title=([^,:=\\s]*)") + } + + private fun getOutputMarcher(macher: Matcher): String? { + return with(macher) { + if (this.find()) { + this.group(1) + } else null + } + } + + fun appendOutputToFile( + tmpLine: String, + workspace: File?, + resultLogFile: String?, + stepId: String? + ) { + val pattenOutput = "::set-output\\s.*" + val prefixOutput = "::set-output " + if (Pattern.matches(pattenOutput, tmpLine)) { + val value = tmpLine.removePrefix(prefixOutput) + + val nameMatcher = getOutputMarcher(OUTPUT_NAME.matcher(value)) ?: "" + val typeMatcher = getOutputMarcher(OUTPUT_TYPE.matcher(value)) ?: "string" // type 默认为string + val labelMatcher = getOutputMarcher(OUTPUT_LABEL.matcher(value)) ?: "" + val pathMatcher = getOutputMarcher(OUTPUT_PATH.matcher(value)) ?: "" + val reportTypeMatcher = getOutputMarcher(OUTPUT_REPORT_TYPE.matcher(value)) ?: "" + + + val keyValue = value.split("::") + if (keyValue.size >= 2) { + // 以逗号为分隔符 左右依次为name type label path reportType + println( + "$nameMatcher," + + "$typeMatcher," + + "$labelMatcher," + + "$pathMatcher," + + "$reportTypeMatcher=${value.removePrefix("${keyValue[0]}::")}\n" + ) + } + } + } + + + private fun appendGateToFile( + tmpLine: String, + list: MutableList + ) { + val pattenOutput = "[\"]?::set-gate-value\\s(.*)" + val prefixOutput = "::set-gate-value " + if (Pattern.matches(pattenOutput, tmpLine)) { + val value = tmpLine.removeSurrounding("\"").removePrefix(prefixOutput) + val name = getOutputMarcher(OUTPUT_NAME.matcher(value)) + val title = getOutputMarcher(OUTPUT_GATE_TITLE.matcher(value)) + val keyValue = value.split("::") + if (keyValue.size >= 2) { + // pass_rate=1,pass_rate=通过率\n + var text = "$name=${value.removePrefix("${keyValue[0]}::")}" + if (!title.isNullOrBlank()) { + text = text.plus(",$name=$title") + } + list.add("$text") + } + } + } + + fun testGateOutPut() { + val testCases = listOf( + "::set-gate-value name=pass_rate::0.9", + "::set-gate-value name=pass_rate,title=测试用例通过率::0.9", + "::set-gate-value name=pass_rate,title=测试全部通过率::0.1", + "::set-gate-value name=errorCount,title=错误总数::100000.00001", + "::set-gate-value name=IntTest,title=整数计数::100", + "::set-gate-value name=isUpload,title=是否上传::true" + ) + val expectResult = listOf( + "pass_rate=0.9", + "pass_rate=0.9,pass_rate=测试用例通过率", + "pass_rate=0.1,pass_rate=测试全部通过率", + "errorCount=100000.00001,errorCount=错误总数", + "IntTest=100,IntTest=整数计数", + "isUpload=true,isUpload=是否上传" + ) + val gateFile = mutableListOf() + testCases.forEach { + appendGateToFile(it, gateFile) + } + checkRealAndExpect(gateFile, expectResult) + val data = mutableMapOf() + val title = mutableMapOf() + gateFile.forEach { + val split = it.split(",") + val nameToValue = split.getOrNull(0) + val nameToTitle = split.getOrNull(1) + keyEqualValueInsertMap(nameToValue, data) + keyEqualValueInsertMap(nameToTitle, title) + } + data.values.forEach { + getQualityDataType(it) + } + // titleCheck + assertEquals("测试全部通过率", title["pass_rate"]) + assertEquals("错误总数", title["errorCount"]) + assertEquals("整数计数", title["IntTest"]) + assertEquals("是否上传", title["isUpload"]) + // dataCheck + assertEquals("0.1", data["pass_rate"]) + assertEquals("true", data["isUpload"]) + assertEquals("100", data["IntTest"]) + } + private fun keyEqualValueInsertMap(nameToValue: String?, map: MutableMap) { + if (nameToValue.isNullOrBlank()) return + val key = nameToValue.split("=").getOrNull(0) ?: throw TaskExecuteException( + errorMsg = "Illegal gateway key set: $nameToValue", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD + ) + val value = nameToValue.split("=").getOrNull(1) ?: throw TaskExecuteException( + errorMsg = "Illegal gateway key set: $nameToValue", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD + ) + map[key] = value.trim() + } + + + private fun checkRealAndExpect(real: MutableList, expect: List) { + real.forEachIndexed { index, _ -> + assertEquals(expect[index], real[index]) + } + } + + private fun getQualityDataType(value: String): QualityDataType { + value.toIntOrNull().let { + if (it != null) { + return QualityDataType.INT + } + } + value.toFloatOrNull().let { + if (it != null) { + return QualityDataType.FLOAT + } + } + if (value == "true" || value == "false") { + return QualityDataType.BOOLEAN + } + throw TaskExecuteException( + errorMsg = "error qualityDataType: $value", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD + ) + } + + fun testOutPut() { + val a = "::set-output name=var_4,type=report,label=测试报告名称,reportType=THIRDPARTY::https://www.xxx.com/" + val b = "::set-output name=var_3,type=report,label=测试报告名称,path=report/::index.html" + val c = "::set-output name=var_2,type=artifact::*.txt" + val d = "::set-output name=var_1::1" + appendOutputToFile(a, null, null, null) + appendOutputToFile(b, null, null, null) + appendOutputToFile(c, null, null, null) + appendOutputToFile(d, null, null, null) + } + + + fun testEnvUtils() { + val command = " echo \${{ ci.workspace }}\n" + + " echo envs.env_a=\${{ envs.env_a }}, env_a=\$env_a\n" + + " echo envs.env_b=\${{ envs.env_b }}, env_b=\$env_b\n" + + " echo envs.env_c=\${{ envs.env_c }}, env_c=\$env_c\n" + + " echo envs.env_d=\${{ envs.env_d }}, env_d=\$env_d\n" + + " echo envs.env_e=\${{ envs.env_e }}, env_e=\$env_e\n" + + " echo envs.a=\${{ envs.a }}, a=\$a\n" + + "\n" + + " echo ::set-output name=a::i am a at step_1" + val data = mapOf("envs.env_a" to "test") +// print(command) + val res = ReplacementUtils.replace( + command = command, + replacement = object : ReplacementUtils.KeyReplacement { + override fun getReplacement(key: String): String? = if (data[key] != null) { + data[key] + } else { + null + } + }, + contextMap = null + ) + assertEquals(res, " echo \\\${{ ci.workspace }}\n" + + " echo envs.env_a=test, env_a=\$env_a\n" + + " echo envs.env_b=\\\${{ envs.env_b }}, env_b=\$env_b\n" + + " echo envs.env_c=\\\${{ envs.env_c }}, env_c=\$env_c\n" + + " echo envs.env_d=\\\${{ envs.env_d }}, env_d=\$env_d\n" + + " echo envs.env_e=\\\${{ envs.env_e }}, env_e=\$env_e\n" + + " echo envs.a=\\\${{ envs.a }}, a=\$a\n" + + "\n" + + " echo ::set-output name=a::i am a at step_1") + }*/ +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/api/QualityApi.kt b/src/main/kotlin/com/tencent/bk/devops/atom/api/QualityApi.kt new file mode 100644 index 0000000..3f694bf --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/api/QualityApi.kt @@ -0,0 +1,43 @@ +package com.tencent.bk.devops.atom.api + +import com.fasterxml.jackson.core.type.TypeReference +import com.tencent.bk.devops.atom.common.QUALITY_ELEMENT_TYPE +import com.tencent.bk.devops.atom.pojo.request.IndicatorCreate +import com.tencent.bk.devops.atom.utils.json.JsonUtil +import com.tencent.bk.devops.plugin.pojo.Result +import okhttp3.RequestBody +import org.slf4j.LoggerFactory + +class QualityApi() : BaseApi() { + + private val logger = LoggerFactory.getLogger(QualityApi::class.java) + + private val urlPrefix = "/quality/api/build" + + fun upsertIndicator( + userId: String, + projectId: String, + indicatorCreate: List + ): Result { + val url = "$urlPrefix/indicator/v3/project/$projectId/upsertIndicator" + val requestBody = RequestBody.create(JSON_CONTENT_TYPE, JsonUtil.toJson(indicatorCreate)) + val request = buildPost(url, requestBody, mutableMapOf("X-DEVOPS-UID" to userId)) + val responseContent = request(request, "创建红线自定义指标失败") + + return JsonUtil.fromJson(responseContent, object : TypeReference>() {}) + } + + fun saveScriptHisMetadata( + taskId: String, + taskName: String, + data: Map + ): Result { + val url = "$urlPrefix/metadata/saveHisMetadata" + + "?elementType=$QUALITY_ELEMENT_TYPE&taskId=$taskId&taskName=$taskName" + val requestBody = RequestBody.create(JSON_CONTENT_TYPE, JsonUtil.toJson(data)) + val request = buildPost(url, requestBody, mutableMapOf()) + val responseContent = request(request, "保存红线数据失败") + + return JsonUtil.fromJson(responseContent, object : TypeReference>() {}) + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/common/Constants.kt b/src/main/kotlin/com/tencent/bk/devops/atom/common/Constants.kt new file mode 100644 index 0000000..2d0e7e7 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/common/Constants.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.common + +const val BUILD_ID = "devops.build.id" + +const val BUILD_TYPE = "build.type" + +const val WORKSPACE_ENV = "WORKSPACE" + +const val WORKSPACE_CONTEXT = "ci.workspace" + +const val CI_TOKEN_CONTEXT = "ci.token" + +const val JOB_OS_CONTEXT = "job.os" + +const val QUALITY_ELEMENT_TYPE = "run" diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/common/ErrorCode.kt b/src/main/kotlin/com/tencent/bk/devops/atom/common/ErrorCode.kt new file mode 100644 index 0000000..2f39b44 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/common/ErrorCode.kt @@ -0,0 +1,19 @@ +package com.tencent.bk.devops.atom.common + +object ErrorCode { + // 插件执行错误 + const val PLUGIN_DEFAULT_ERROR = 2199001 // 插件异常默认 + const val PLUGIN_CREATE_QUALITY_INDICATOR_ERROR = 2199002 + const val PLUGIN_SAVE_QUALITY_DATA_ERROR = 2199003 + // 用户使用错误 + const val USER_INPUT_INVAILD = 2199002 // 用户输入数据有误 + const val USER_RESOURCE_NOT_FOUND = 2199003 // 找不到对应系统资源 + const val USER_TASK_OPERATE_FAIL = 2199004 // 插件执行过程出错 + const val USER_JOB_OUTTIME_LIMIT = 2199005 // 用户Job排队超时（自行限制） + const val USER_TASK_OUTTIME_LIMIT = 2199006 // 用户插件执行超时（自行限制） + const val USER_QUALITY_CHECK_FAIL = 2199007 // 质量红线检查失败 + const val USER_QUALITY_REVIEW_ABORT = 2199008 // 质量红线审核驳回 + const val USER_SCRIPT_COMMAND_INVAILD = 2199009 // 脚本命令无法正常执行 + const val USER_STAGE_FASTKILL_TERMINATE = 2199010 // 因用户配置了FastKill导致的终止执行 + const val USER_SCRIPT_TASK_FAIL = 2199011 // bash脚本发生用户错误 +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/common/RunEnvHelper.kt b/src/main/kotlin/com/tencent/bk/devops/atom/common/RunEnvHelper.kt new file mode 100644 index 0000000..06f4556 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/common/RunEnvHelper.kt @@ -0,0 +1,156 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.common + +import com.fasterxml.jackson.core.type.TypeReference +import com.tencent.bk.devops.atom.pojo.StringData +import com.tencent.bk.devops.atom.utils.http.SdkUtils +import com.tencent.bk.devops.plugin.utils.JsonUtil +// import com.tencent.devops.api.CodeccApi +// import com.tencent.devops.pojo.BuildScriptType +// import com.tencent.devops.pojo.CodeccCheckAtomParam +// import com.tencent.devops.pojo.CodeccCheckAtomParamV3 +// import com.tencent.devops.pojo.CodeccExecuteConfig +// import com.tencent.devops.pojo.OSType +// import com.tencent.devops.utils.common.AgentEnv +import java.io.File + +object RunEnvHelper { + +// private val api = CodeccApi() + + private val ENV_FILES = arrayOf("result.log", "result.ini") + + private var codeccWorkspace: File? = null + + init { + // 第三方构建机 +// if (getOS().isThirdParty()) { +// println("[初始化] 检测到这是第三方构建机") +// } + } + + fun getRunEnv(workspace: String): Map { + val result = mutableMapOf() + ENV_FILES.forEach { result.putAll(readScriptEnv(File(workspace), it)) } + return result + } + + private fun readScriptEnv(workspace: File, file: String): Map { + val f = File(workspace, file) + if (!f.exists()) { + return mapOf() + } + if (f.isDirectory) { + return mapOf() + } + + val lines = f.readLines() + if (lines.isEmpty()) { + return mapOf() + } + // KEY-VALUE + return lines.filter { it.contains("=") }.map { + val split = it.split("=", ignoreCase = false, limit = 2) + split[0].trim() to StringData(split[1].trim()) + }.toMap() + } + +// fun saveTask(atomContext: AtomContext) { +// with(atomContext.param) { +// api.saveTask(projectName, pipelineId, pipelineBuildId) +// } +// } + +// fun getScriptType(): BuildScriptType { +// return when (getOS()) { +// OSType.MAC_OS, OSType.LINUX -> BuildScriptType.SHELL +// OSType.WINDOWS -> BuildScriptType.BAT +// else -> BuildScriptType.SHELL +// } +// } + +// fun getOS(): OSType { +// val osName = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH) +// return if (osName.indexOf(string = "mac") >= 0 || osName.indexOf("darwin") >= 0) { +// OSType.MAC_OS +// } else if (osName.indexOf("win") >= 0) { +// OSType.WINDOWS +// } else if (osName.indexOf("nux") >= 0) { +// OSType.LINUX +// } else { +// OSType.OTHER +// } +// } + + fun getVariable(): Map { + val map = JsonUtil.to(File(SdkUtils.getInputFile()).readText(), object : TypeReference>() {}) + return map.map { it.key to it.value.toString() }.toMap() + } + +// // 第三方构建机初始化 +// fun thirdInit(codeccExecuteConfig: CodeccExecuteConfig) { +// // 第三方构建机安装环境 +// if (AgentEnv.isThirdParty()) { +// when (getOS()) { +// OSType.LINUX -> { +// CodeccInstaller.setUpPython3(codeccExecuteConfig.atomContext.param) +// } +// else -> { +// } +// } +// } +// } + +// fun getCodeccWorkspace(param: CodeccCheckAtomParamV3): File { +// if (codeccWorkspace != null) return codeccWorkspace!! +// +// val workspace = File(param.bkWorkspace) +// val buildId = param.pipelineBuildId +// +// // Copy the nfs coverity file to workspace +// println("[初始化] get the workspace: ${workspace.canonicalPath}") +// println("[初始化] get the workspace parent: ${workspace.parentFile?.canonicalPath} | '${File.separatorChar}'") +// println("[初始化] get the workspace parent string: ${workspace.parent}") +// +// val tempDir = File(workspace, ".temp") +// println("[初始化] get the workspace path parent: ${tempDir.canonicalPath}") +// codeccWorkspace = File(tempDir, "codecc_$buildId") +// if (!codeccWorkspace!!.exists()) { +// codeccWorkspace!!.mkdirs() +// } +// +// return codeccWorkspace!! +// } + + fun deleteCodeccWorkspace() { + println("delete the workspace path parent: $codeccWorkspace") + if (codeccWorkspace != null) { + codeccWorkspace!!.deleteOnExit() + } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/common/utils/ReplacementUtils.kt b/src/main/kotlin/com/tencent/bk/devops/atom/common/utils/ReplacementUtils.kt new file mode 100644 index 0000000..e3c296d --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/common/utils/ReplacementUtils.kt @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.common.utils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +object ReplacementUtils { + + fun replace(command: String, replacement: KeyReplacement): String { + return replace(command, replacement, emptyMap()) + } + + fun replace( + command: String, + replacement: KeyReplacement, + contextMap: Map? = emptyMap() + ): String { + if (command.isBlank()) { + return command + } + val sb = StringBuilder() + + val lines = command.lines() + lines.forEachIndexed { index, line -> + // 忽略注释 + val template = if (line.trim().startsWith("#")) { + line + } else { + parseTemplate(line, replacement, contextMap) + } + sb.append(template) + if (index != lines.size - 1) { + sb.append("\n") + } + } + return sb.toString() + } + + private fun parseTemplate( + command: String, + replacement: KeyReplacement, + contextMap: Map?, + depth: Int = 1 + ): String { + if (depth < 0) { + return command + } + val matcher = tPattern.matcher(command) + val buff = StringBuffer() + while (matcher.find()) { + val key = (matcher.group("single") ?: matcher.group("double")).trim() + var value = replacement.getReplacement(key) ?: contextMap?.get(key) + if (value == null) { + // 修复错误的替换（bad substitution）错误 + if (matcher.group().startsWith("\${{")) { + value = "\\${matcher.group()}" + } else { + value = matcher.group() + } + } else { + if (depth > 0 && tPattern.matcher(value).find()) { + value = parseTemplate(value, replacement, contextMap, depth = depth - 1) + } + } + matcher.appendReplacement(buff, Matcher.quoteReplacement(value)) + } + matcher.appendTail(buff) + return if (buff.isNotEmpty()) buff.toString() else command + } + + private val tPattern = Pattern.compile("(\\$[{](?[^$^{}]+)})|(\\$[{]{2}(?[^$^{}]+)[}]{2})") + + interface KeyReplacement { + /** + * 如果[key]替换不成功需要返回null，不建议直接返回[key]，避免无法判断到底替换成功 + */ + fun getReplacement(key: String): String? + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/enums/CharsetType.kt b/src/main/kotlin/com/tencent/bk/devops/atom/enums/CharsetType.kt new file mode 100644 index 0000000..9e4c3aa --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/enums/CharsetType.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.enums + +enum class CharsetType { + /*默认类型*/ + DEFAULT, + /*UTF_8*/ + UTF_8, + /*GBK*/ + GBK +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/enums/OSType.kt b/src/main/kotlin/com/tencent/bk/devops/atom/enums/OSType.kt new file mode 100644 index 0000000..275093e --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/enums/OSType.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.enums + +/** + * + * Powered By Tencent + */ +enum class OSType { + /*WINDOWS系统*/ + WINDOWS, + /*LINUX系统*/ + LINUX, + /*MAC_OS系统*/ + MAC_OS, + /*其他系统*/ + OTHER +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/pojo/AdditionalOptions.kt b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/AdditionalOptions.kt new file mode 100644 index 0000000..25e1965 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/AdditionalOptions.kt @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.pojo + +import com.tencent.bk.devops.atom.enums.OSType +import com.tencent.bk.devops.atom.exception.AtomException + +data class AdditionalOptions( + var shell: ShellType +) { + constructor(shell: String) : this(ShellType.BASH) { + /*用户没有配置脚本类型就按照系统类型默认选择*/ + this.shell = if (shell.isNullOrBlank()) { + ShellType.get(AgentEnv.getOS()) + } else { + ShellType.get(shell) + } + } +} + +enum class ShellType(val shellName: String) { + /*bash*/ + BASH("bash"), + + /*cmd*/ + CMD("cmd"), + + /*powershell*/ + POWERSHELL_CORE("pwsh"), + + /*powershell*/ + POWERSHELL_DESKTOP("powershell"), + + /*python*/ + PYTHON("python"), + + /*sh命令*/ + SH("sh"), + + /*按系统默认*/ + AUTO("auto"); + + companion object { + fun get(value: String): ShellType { + if (value == AUTO.shellName) { + return get(AgentEnv.getOS()) + } + values().forEach { + if (value == it.shellName) return it + } + throw AtomException( + "The current system(${AgentEnv.getOS()}) not support $value yet" + ) + } + + fun get(value: OSType): ShellType { + return when { + value == OSType.WINDOWS -> CMD + value == OSType.LINUX -> BASH + else -> BASH + } + } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/pojo/AgentEnv.kt b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/AgentEnv.kt new file mode 100644 index 0000000..d6ef328 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/AgentEnv.kt @@ -0,0 +1,34 @@ +package com.tencent.bk.devops.atom.pojo + +import com.tencent.bk.devops.atom.enums.OSType +import org.slf4j.LoggerFactory +import java.util.Locale + +object AgentEnv { + + private val logger = LoggerFactory.getLogger(AgentEnv::class.java) + + private var os: OSType? = null + + /*获取系统类型*/ + fun getOS(): OSType { + if (os == null) { + synchronized(this) { + if (os == null) { + val os = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH) + logger.info("Get the os name - ($os)") + this.os = if (os.indexOf(string = "mac") >= 0 || os.indexOf("darwin") >= 0) { + OSType.MAC_OS + } else if (os.indexOf("win") >= 0) { + OSType.WINDOWS + } else if (os.indexOf("nux") >= 0) { + OSType.LINUX + } else { + OSType.OTHER + } + } + } + } + return os!! + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/pojo/BuildEnv.kt b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/BuildEnv.kt new file mode 100644 index 0000000..2edea29 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/BuildEnv.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.pojo + +data class BuildEnv( + val name: String, + val version: String, + val binPath: String, + val env: Map +) diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/pojo/ScriptRunAtomParam.kt b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/ScriptRunAtomParam.kt new file mode 100644 index 0000000..cf63499 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/ScriptRunAtomParam.kt @@ -0,0 +1,22 @@ +package com.tencent.bk.devops.atom.pojo + +import com.fasterxml.jackson.annotation.JsonProperty +import lombok.Data +import lombok.EqualsAndHashCode + +/** + * 插件参数定义 + * @version 1.0 + */ +@Data +@EqualsAndHashCode(callSuper = true) +class ScriptRunAtomParam : AtomBaseParam() { + + /*脚本内容*/ + val script: String = "" + /*字符集类型*/ + @JsonProperty("charsetType") + val charSetType: String = "" + /*脚本类型*/ + val shell: String = "" +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/pojo/request/IndicatorCreate.kt b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/request/IndicatorCreate.kt new file mode 100644 index 0000000..d54db84 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/pojo/request/IndicatorCreate.kt @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.pojo.request + +data class IndicatorCreate( + val name: String, + val cnName: String = "", + val desc: String = "", + val dataType: QualityDataType, + val operation: List = listOf(), + val threshold: String = "", + val elementType: String = "" +) + +enum class QualityDataType { + INT, + BOOLEAN, + FLOAT, + STRING +} + +enum class QualityOperation { + GT, + GE, + LT, + LE, + EQ; + + companion object { + fun convertToSymbol(operation: QualityOperation): String { + return when (operation) { + GT -> ">" + GE -> ">=" + LT -> "<" + LE -> "<=" + EQ -> "=" + } + } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommandLineExecutor.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommandLineExecutor.kt new file mode 100644 index 0000000..654a5f1 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommandLineExecutor.kt @@ -0,0 +1,194 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import org.apache.commons.exec.CommandLine +import org.apache.commons.exec.DefaultExecutor +import org.apache.commons.exec.ExecuteStreamHandler +import org.apache.commons.exec.Executor +import org.slf4j.LoggerFactory +import java.io.File +import java.io.IOException + +@Suppress("ALL") +class CommandLineExecutor : DefaultExecutor() { + + /** the first exception being caught to be thrown to the caller */ + private var exceptionCaught: IOException? = null + + override fun execute(command: CommandLine, environment: MutableMap?): Int { + if (workingDirectory != null && !workingDirectory.exists()) { + throw IOException("$workingDirectory doesn't exist.") + } + + return executeInternal(command, environment, workingDirectory, streamHandler) + } + + /** + * Execute an internal process. If the executing thread is interrupted while waiting for the + * child process to return the child process will be killed. + * + * @param command the command to execute + * @param environment the execution environment + * @param dir the working directory + * @param streams process the streams (in, out, err) of the process + * @return the exit code of the process + * @throws IOException executing the process failed + */ + private fun executeInternal( + command: CommandLine, + environment: Map?, + dir: File, + streams: ExecuteStreamHandler + ): Int { + + setExceptionCaught(null) + + val process = this.launch(command, environment, dir) + + try { + streams.setProcessInputStream(process.outputStream) + streams.setProcessOutputStream(process.inputStream) + streams.setProcessErrorStream(process.errorStream) + } catch (e: IOException) { + process.destroy() + throw e + } + + streams.start() + try { + + // add the process to the list of those to destroy if the VM exits + if (this.processDestroyer != null) { + this.processDestroyer.add(process) + } + + // associate the watchdog with the newly created process + if (watchdog != null) { + watchdog.start(process) + } + + var exitValue = Executor.INVALID_EXITVALUE + + try { + exitValue = process.waitFor() + } catch (e: InterruptedException) { + process.destroy() + } finally { + // see http://bugs.sun.com/view_bug.do?bug_id=6420270 + // see https://issues.apache.org/jira/browse/EXEC-46 + // Process.waitFor should clear interrupt status when throwing InterruptedException + // but we have to do that manually + Thread.interrupted() + } + + if (watchdog != null) { + watchdog.stop() + } + + try { + streams.stop() + } catch (e: IOException) { + setExceptionCaught(e) + } + + closeProcessStreams(process) + + if (getExceptionCaught() != null) { + throw getExceptionCaught()!! + } + + if (watchdog != null) { + try { + watchdog.checkException() + } catch (e: IOException) { + throw e + } catch (e: Exception) { + throw IOException(e.message) + } + } + + return exitValue + } finally { + // remove the process to the list of those to destroy if the VM exits + if (this.processDestroyer != null) { + this.processDestroyer.remove(process) + } + } + } + + /** + * Close the streams belonging to the given Process. + * + * @param process the Process. + */ + private fun closeProcessStreams(process: Process) { + + try { + process.inputStream.close() + } catch (e: IOException) { + setExceptionCaught(e) + } + + try { + process.outputStream.close() + } catch (e: IOException) { + setExceptionCaught(e) + } + + try { + process.errorStream.close() + } catch (e: IOException) { + setExceptionCaught(e) + } + } + + /** + * Keep track of the first IOException being thrown. + * + * @param e the IOException + */ + private fun setExceptionCaught(e: IOException?) { + if (this.exceptionCaught == null) { + this.exceptionCaught = e + } + } + + /** + * Get the first IOException being thrown. + * + * @return the first IOException being caught + */ + private fun getExceptionCaught(): IOException? { + return this.exceptionCaught + } + + companion object { + private val logger = LoggerFactory.getLogger(CommandLineExecutor::class.java) + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommandLineUtils.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommandLineUtils.kt new file mode 100644 index 0000000..6df34ea --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommandLineUtils.kt @@ -0,0 +1,286 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.exception.AtomException +import org.apache.commons.exec.CommandLine +import org.apache.commons.exec.LogOutputStream +import org.apache.commons.exec.PumpStreamHandler +import org.slf4j.LoggerFactory +import java.io.ByteArrayOutputStream +import java.io.File +import java.nio.charset.Charset +import java.util.regex.Matcher +import java.util.regex.Pattern + +object CommandLineUtils { + + private val logger = LoggerFactory.getLogger(CommandLineUtils::class.java) + /*OUTPUT_NAME 正则匹配规则*/ + private val OUTPUT_NAME = Pattern.compile("name=([^,:=\\s]*)") + /*OUTPUT_TYPE 正则匹配规则*/ + private val OUTPUT_TYPE = Pattern.compile("type=([^,:=\\s]*)") + /*OUTPUT_LABEL 正则匹配规则*/ + private val OUTPUT_LABEL = Pattern.compile("label=([^,:=\\s]*)") + /*OUTPUT_PATH 正则匹配规则*/ + private val OUTPUT_PATH = Pattern.compile("path=([^,:=\\s]*)") + /*OUTPUT_REPORT_TYPE 正则匹配规则*/ + private val OUTPUT_REPORT_TYPE = Pattern.compile("reportType=([^,:=\\s]*)") + /*OUTPUT_GATE_TITLE 正则匹配规则*/ + private val OUTPUT_GATE_TITLE = Pattern.compile("title=([^,:=\\s]*)") + + private val lineParser = listOf(OauthCredentialLineParser()) + + fun execute( + command: String, + workspace: File?, + print2Logger: Boolean, + prefix: String = "", + executeErrorMessage: String? = null, + buildId: String, + stepId: String? = null, + charSetType: CharsetType? = null + ): String { + /*result 用于装载返回信息*/ + val result = StringBuilder() + logger.debug("will execute command >>> $command") + + /*解析命令*/ + val cmdLine = CommandLine.parse(command) + /*生成executor*/ + val executor = CommandLineExecutor() + if (workspace != null) { + /*工作空间已知则装载进去*/ + executor.workingDirectory = workspace + } + /*获取上下文文件*/ + val contextLogFile = if (buildId.isNotBlank()) { + ScriptEnvUtils.getContextFile(buildId) + } else { + null + } + + /*获取字符集编码类型*/ + val charset = when (charSetType) { + CharsetType.UTF_8 -> "UTF-8" + CharsetType.GBK -> "GBK" + else -> Charset.defaultCharset().name() + } + /*定义output标准输出流*/ + val outputStream = object : LogOutputStream() { + override fun processBuffer() { + val privateStringField = LogOutputStream::class.java.getDeclaredField("buffer") + privateStringField.isAccessible = true + /*反射拿到buffer 解决字符集编码问题*/ + val buffer = privateStringField.get(this) as ByteArrayOutputStream + processLine(buffer.toString(charset)) + /*手动reset*/ + buffer.reset() + } + + override fun processLine(line: String?, level: Int) { + if (line == null) + return + + /*补齐前缀*/ + var tmpLine: String = prefix + line + + lineParser.forEach { + /*做日志脱敏*/ + tmpLine = it.onParseLine(tmpLine) + } + if (print2Logger) { + /*提取特殊内容到文件进行持久化存储并输出到上下文*/ + appendResultToFile(executor.workingDirectory, contextLogFile, tmpLine) + } + println(tmpLine) + /*装载result*/ + result.append(tmpLine).append("\n") + } + } + /*定义error输出流*/ + val errStream = object : LogOutputStream() { + override fun processBuffer() { + val privateStringField = LogOutputStream::class.java.getDeclaredField("buffer") + privateStringField.isAccessible = true + /*反射拿到buffer 解决字符集编码问题*/ + val buffer = privateStringField.get(this) as ByteArrayOutputStream + processLine(buffer.toString(charset)) + /*手动reset*/ + buffer.reset() + } + + override fun processLine(line: String?, level: Int) { + if (line == null) + return + + /*补齐前缀*/ + var tmpLine: String = prefix + line + + lineParser.forEach { + /*做日志脱敏*/ + tmpLine = it.onParseLine(tmpLine) + } + if (print2Logger) { + /*提取特殊内容到文件进行持久化存储并输出到上下文*/ + appendResultToFile(executor.workingDirectory, contextLogFile, tmpLine) + } + logger.error(tmpLine) + /*装载result*/ + result.append(tmpLine).append("\n") + } + } + /*定义好输出流*/ + executor.streamHandler = PumpStreamHandler(outputStream, errStream) + try { + /*执行脚本*/ + val exitCode = executor.execute(cmdLine) + if (exitCode != 0) { + /*执行返回码，非零表示执行出错，这时直接抛错。为用户自己的脚本问题*/ + throw AtomException( + "$prefix Script command execution failed with exit code($exitCode)" + ) + } + } catch (ignored: Throwable) { + /*对其余异常兜底处理，可能是执行脚本时抛错的错。*/ + val errorMessage = executeErrorMessage ?: "Fail to execute the command($command)" + logger.warn(errorMessage) + throw AtomException( + ignored.message ?: "" + ) + } + return result.toString() + } + + /*写内容到文件*/ + private fun appendResultToFile( + workspace: File?, + resultLogFile: String?, + tmpLine: String + ) { + /*写入红线指标信息*/ + parseGate(tmpLine)?.let { + File(workspace, ScriptEnvUtils.getQualityGatewayEnvFile()).appendText(it + "\n") + } + + if (resultLogFile == null) { + return + } + /*写variable到文件*/ + parseVariable(tmpLine)?.let { + File(workspace, resultLogFile).appendText(it + "\n") + } + /*写output到文件*/ + parseOutput(tmpLine)?.let { + File(workspace, resultLogFile).appendText(it + "\n") + } + } + + /*解析variable格式的变量*/ + fun parseVariable( + tmpLine: String + ): String? { + /*相应匹配规则*/ + val pattenVar = "[\"]?::set-variable\\sname=.*" + val prefixVar = "::set-variable name=" + if (Pattern.matches(pattenVar, tmpLine)) { + /*正则匹配后做拆分处理*/ + val value = tmpLine.removeSurrounding("\"").removePrefix(prefixVar) + val keyValue = value.split("::") + if (keyValue.size >= 2) { + return "variables.${keyValue[0]}=${value.removePrefix("${keyValue[0]}::")}" + } + } + return null + } + + fun parseOutput( + tmpLine: String + ): String? { + /*相应匹配规则*/ + val pattenOutput = "[\"]?::set-output\\s(.*)" + val prefixOutput = "::set-output " + if (Pattern.matches(pattenOutput, tmpLine)) { + /*正则匹配后做拆分处理*/ + val value = tmpLine.removeSurrounding("\"").removePrefix(prefixOutput) + + val nameMatcher = getOutputMarcher(OUTPUT_NAME.matcher(value)) ?: "" + val typeMatcher = getOutputMarcher(OUTPUT_TYPE.matcher(value)) ?: "string" // type 默认为string + val labelMatcher = getOutputMarcher(OUTPUT_LABEL.matcher(value)) ?: "" + val pathMatcher = getOutputMarcher(OUTPUT_PATH.matcher(value)) ?: "" + val reportTypeMatcher = getOutputMarcher(OUTPUT_REPORT_TYPE.matcher(value)) ?: "" + + /*对5种类型的标志位分别存储，互不干扰*/ + val keyValue = value.split("::") + if (keyValue.size >= 2) { + // 以逗号为分隔符 左右依次为name type label path reportType + return "$nameMatcher," + + "$typeMatcher," + + "$labelMatcher," + + "$pathMatcher," + + "$reportTypeMatcher=${value.removePrefix("${keyValue[0]}::")}" + } + } + return null + } + + fun parseGate( + tmpLine: String + ): String? { + /*相应匹配规则*/ + val pattenOutput = "[\"]?::set-gate-value\\s(.*)" + val prefixOutput = "::set-gate-value " + if (Pattern.matches(pattenOutput, tmpLine)) { + /*正则匹配后做拆分处理*/ + val value = tmpLine.removeSurrounding("\"").removePrefix(prefixOutput) + val name = getOutputMarcher(OUTPUT_NAME.matcher(value)) + val title = getOutputMarcher(OUTPUT_GATE_TITLE.matcher(value)) + /*对2种类型的标志位分别存储，互不干扰*/ + val keyValue = value.split("::") + if (keyValue.size >= 2) { + // pass_rate=1,pass_rate=通过率\n + var text = "$name=${value.removePrefix("${keyValue[0]}::")}" + if (!title.isNullOrBlank()) { + text = text.plus(",$name=$title") + } + return text + } + } + return null + } + + private fun getOutputMarcher(macher: Matcher): String? { + return with(macher) { + /*只返回匹配到的第一个，否则返回null*/ + if (this.find()) { + this.group(1) + } else null + } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommonEnv.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommonEnv.kt new file mode 100644 index 0000000..8a342fb --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommonEnv.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import org.slf4j.LoggerFactory + +// import com.tencent.devops.worker.common.logger.LoggerService + +/** + * This is to store the common build env + */ +object CommonEnv { + + private val logger = LoggerFactory.getLogger(CommonEnv::class.java) + + private val envMap = HashMap() + private var svnUser: String? = null + private var svnPass: String? = null + + var fileDevnetGateway: String? = null + var fileIdcGateway: String? = null + + fun addCommonEnv(env: Map) { + logger.info("Add the env($env) to common environment") + envMap.putAll(env) + } + + fun getCommonEnv(): Map { + return HashMap(envMap) + } + + fun addSvnHttpCredential(user: String, password: String) { + svnUser = user + svnPass = password + } + + fun getSvnHttpCredential(): Pair? { + if (svnUser.isNullOrBlank() || svnPass.isNullOrBlank()) { + return null + } + return Pair(svnUser!!, svnPass!!) + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommonUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommonUtil.kt new file mode 100644 index 0000000..0ff7615 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/CommonUtil.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import org.slf4j.LoggerFactory +import java.io.File + +object CommonUtil { + + private val logger = LoggerFactory.getLogger(CommonUtil::class.java) + + /*统一打日志，debug信息用于问题排查*/ + fun printTempFileInfo(file: File) { + logger.debug("--------file(${file.name}) debug info-------------") + logger.debug("absolutePath: ${file.absolutePath}") + logger.debug("Size: ${file.length()}") + logger.debug("canExecute/canRead/canWrite: ${file.canExecute()}/${file.canRead()}/${file.canWrite()}") + logger.debug("--------file debug info end-------------") + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/Constants.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/Constants.kt new file mode 100644 index 0000000..320458a --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/Constants.kt @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +/*getEnvironmentPathPrefix 公共方法*/ +fun getEnvironmentPathPrefix(): String { + val os = System.getProperty("os.name") + if (os.isNullOrEmpty()) { + return ENVIRONMENT_LINUX_PATH_PREFIX + } + if (os.startsWith("mac", true)) { + return ENVIRONMENT_MAC_PATH_PREFIX + } + return ENVIRONMENT_LINUX_PATH_PREFIX +} + +/*LINUX前缀*/ +private const val ENVIRONMENT_LINUX_PATH_PREFIX = "/data/bkdevops/apps/" + +/*MAC前缀*/ +private const val ENVIRONMENT_MAC_PATH_PREFIX = "/Users/bkdevops/apps/" diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/ExecutorUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/ExecutorUtil.kt new file mode 100644 index 0000000..12f396e --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/ExecutorUtil.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +object ExecutorUtil { + + private val threadLocal = ThreadLocal() + + /*设置线程本地变量*/ + private fun setThreadLocal() { + val randomNum = UUIDUtil.generate() + threadLocal.set(randomNum) + } + + /*同线程会获取相同的变量*/ + fun getThreadLocal(): String { + val value = threadLocal.get() + if (value == null) { + setThreadLocal() + } + return threadLocal.get() + } + + /*删除线程变量*/ + fun removeThreadLocal() { + threadLocal.remove() + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/OauthCredentialLineParser.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/OauthCredentialLineParser.kt new file mode 100644 index 0000000..68ada4c --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/OauthCredentialLineParser.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import org.slf4j.LoggerFactory +import java.util.regex.Pattern + +/*oauth 凭证处理*/ +class OauthCredentialLineParser { + fun onParseLine(line: String): String { + if (line.contains("http://oauth2:")) { + val pattern = Pattern.compile("oauth2:(\\w+)@") + val matcher = pattern.matcher(line) + /*脱敏*/ + val replace = matcher.replaceAll("") + logger.info("Parse the line from $line to $replace") + return replace + } + return line + } + + companion object { + private val logger = LoggerFactory.getLogger(OauthCredentialLineParser::class.java) + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/ScriptEnvUtils.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/ScriptEnvUtils.kt new file mode 100644 index 0000000..b7db6fd --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/ScriptEnvUtils.kt @@ -0,0 +1,178 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import org.slf4j.LoggerFactory +import java.io.File + +object ScriptEnvUtils { + private const val ENV_FILE = "result.log" + private const val MULTILINE_FILE = "multiLine.log" + private const val CONTEXT_FILE = "context.log" + private const val QUALITY_GATEWAY_FILE = "gatewayValueFile.ini" + private val keyRegex = Regex("^[a-zA-Z_][a-zA-Z0-9_]*$") + private val logger = LoggerFactory.getLogger(ScriptEnvUtils::class.java) + + fun cleanEnv(buildId: String, workspace: File) { + cleanScriptEnv(workspace, getEnvFile(buildId)) + cleanScriptEnv(workspace, getDefaultEnvFile(buildId)) + } + + fun cleanContext(buildId: String, workspace: File) { + cleanScriptEnv(workspace, getContextFile(buildId)) + } + + /*通过env文件，获取到环境变量map*/ + fun getEnv(buildId: String, workspace: File): Map { + return readScriptEnv(workspace, "$buildId-$ENV_FILE") + .plus(readScriptEnv(workspace, getEnvFile(buildId))) + } + + /*通过上下文文件，获取到上下文内容*/ + fun getContext(buildId: String, workspace: File): Map { + return readScriptContext(workspace, getContextFile(buildId)) + } + + /*限定文件名*/ + fun getEnvFile(buildId: String): String { + val randomNum = ExecutorUtil.getThreadLocal() + return "$buildId-$randomNum-$ENV_FILE" + } + + /*获取多行内容*/ + fun getMultipleLines(buildId: String, workspace: File): List { + return readLines(workspace, getMultipleLineFile(buildId)) + } + + /*限定文件名*/ + fun getMultipleLineFile(buildId: String): String { + val randomNum = ExecutorUtil.getThreadLocal() + return "$buildId-$randomNum-$MULTILINE_FILE" + } + + /*限定文件名*/ + fun getContextFile(buildId: String): String { + val randomNum = ExecutorUtil.getThreadLocal() + return "$buildId-$randomNum-$CONTEXT_FILE" + } + + /*限定文件名*/ + private fun getDefaultEnvFile(buildId: String): String { + return "$buildId-$ENV_FILE" + } + + /*清理逻辑*/ + fun cleanWhenEnd(buildId: String, workspace: File) { + /*获取文件路径*/ + val defaultEnvFilePath = getDefaultEnvFile(buildId) + val randomEnvFilePath = getEnvFile(buildId) + val randomContextFilePath = getContextFile(buildId) + val multiLineFilePath = getMultipleLineFile(buildId) + /*清理文件*/ + deleteFile(multiLineFilePath, workspace) + deleteFile(defaultEnvFilePath, workspace) + deleteFile(randomEnvFilePath, workspace) + deleteFile(randomContextFilePath, workspace) + /*销毁线程临时变量数据*/ + ExecutorUtil.removeThreadLocal() + } + + /*清理文件逻辑*/ + private fun deleteFile(filePath: String, workspace: File) { + val defaultFile = File(workspace, filePath) + if (defaultFile.exists()) { + defaultFile.delete() + } + } + + fun getQualityGatewayEnvFile() = QUALITY_GATEWAY_FILE + + /*清理文件*/ + private fun cleanScriptEnv(workspace: File, file: String) { + val scriptFile = File(workspace, file) + if (scriptFile.exists()) { + scriptFile.delete() + } + if (!scriptFile.createNewFile()) { + logger.warn("Fail to create the file - (${scriptFile.absolutePath})") + } else { + scriptFile.deleteOnExit() + } + } + + /*读取env文件*/ + private fun readScriptEnv(workspace: File, file: String): Map { + val f = File(workspace, file) + if (!f.exists() || f.isDirectory) { + return mapOf() + } + /*读取文件内容*/ + val lines = f.readLines() + return if (lines.isEmpty()) { + mapOf() + } else { + // KEY-VALUE 格式读取 + lines.filter { it.contains("=") }.map { + val split = it.split("=", ignoreCase = false, limit = 2) + split[0].trim() to split[1].trim() + }.filter { + // #3453 保存时再次校验key的合法性 + keyRegex.matches(it.first) + }.toMap() + } + } + + private fun readLines(workspace: File, file: String): List { + val f = File(workspace, file) + /*不存在或是文件夹则返回空。非预期情况*/ + if (!f.exists() || f.isDirectory) { + return emptyList() + } + return f.readLines() + } + + /*读取上下文文件*/ + private fun readScriptContext(workspace: File, file: String): Map { + val f = File(workspace, file) + if (!f.exists() || f.isDirectory) { + return mapOf() + } + + /*读取文件内容*/ + val lines = f.readLines() + return if (lines.isEmpty()) { + mapOf() + } else { + // KEY-VALUE + lines.filter { it.contains("=") }.map { + val split = it.split("=", ignoreCase = false, limit = 2) + split[0].trim() to split[1].trim() + }.toMap() + } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/SensitiveLineParser.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/SensitiveLineParser.kt new file mode 100644 index 0000000..14cbc36 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/SensitiveLineParser.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import org.slf4j.LoggerFactory +import java.util.regex.Pattern + +object SensitiveLineParser { + private val pattern = Pattern.compile("oauth2:(\\w+)@") + private val patternPassword = Pattern.compile("http://.*:.*@") + + fun onParseLine(line: String): String { + if (line.contains("http://oauth2:")) { + val matcher = pattern.matcher(line) + val replace = matcher.replaceAll("oauth2:***@") + logger.info("Parse the line from $line to $replace") + return replace + } + if (line.contains("http://")) { + return patternPassword.matcher(line).replaceAll("http://***:***@") + } + return line + } + + private val logger = LoggerFactory.getLogger(SensitiveLineParser::class.java) +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/UUIDUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/UUIDUtil.kt new file mode 100644 index 0000000..78f60a9 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/UUIDUtil.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils + +import java.util.UUID + +/** + * + * Powered By Tencent + */ +object UUIDUtil { + /** + * 生成32位字符随机UUID + * @return UUID字符串 + */ + fun generate(): String { + val uuid = UUID.randomUUID() + val str = uuid.toString() + // 去掉"-"符号 + return str.substring(0, 8) + str.substring(9, 13) + str.substring(14, 18) + str.substring( + 19, + 23 + ) + str.substring(24) + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/BashUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/BashUtil.kt new file mode 100644 index 0000000..2c72cd3 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/BashUtil.kt @@ -0,0 +1,282 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils.script + +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.enums.OSType +import com.tencent.bk.devops.atom.exception.AtomException +import com.tencent.bk.devops.atom.pojo.AgentEnv +import com.tencent.bk.devops.atom.pojo.BuildEnv +import com.tencent.bk.devops.atom.utils.CommandLineUtils +import com.tencent.bk.devops.atom.utils.CommonUtil +import com.tencent.bk.devops.atom.utils.ScriptEnvUtils +import com.tencent.bk.devops.atom.utils.getEnvironmentPathPrefix +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.charset.Charset +import java.nio.file.Files + +@Suppress("ALL") +object BashUtil { + + // + private const val setEnv = "setEnv(){\n" + + " local key=\$1\n" + + " local val=\$2\n" + + "\n" + + " if [[ -z \"\$@\" ]]; then\n" + + " return 0\n" + + " fi\n" + + "\n" + + " if ! echo \"\$key\" | grep -qE \"^[a-zA-Z_][a-zA-Z0-9_]*\$\"; then\n" + + " echo \"[\$key] is invalid\" >&2\n" + + " return 1\n" + + " fi\n" + + "\n" + + " echo \$key=\$val >> ##resultFile##\n" + + " export \$key=\"\$val\"\n" + + " }\n" + private const val format_multiple_lines = "format_multiple_lines() {\n" + + " local content=\$1\n" + + " content=\"\${content//'%'/'%25'}\"\n" + + " content=\"\${content//\$'\\n'/'%0A'}\"\n" + + " content=\"\${content//\$'\\r'/'%0D'}\"\n" + + " /bin/echo \"\$content\"|sed 's/\\\\n/%0A/g'|sed 's/\\\\r/%0D/g' >> ##resultFile##\n" + + "}\n" + // +// private const val setGateValue = "setGateValue(){\n" + +// " local key=\$1\n" + +// " local val=\$2\n" + +// "\n" + +// " if [[ -z \"\$@\" ]]; then\n" + +// " return 0\n" + +// " fi\n" + +// "\n" + +// " if ! echo \"\$key\" | grep -qE \"^[a-zA-Z_][a-zA-Z0-9_]*\$\"; then\n" + +// " echo \"[\$key] is invalid\" >&2\n" + +// " return 1\n" + +// " fi\n" + +// "\n" + +// " echo \$key=\$val >> ##gateValueFile##\n" + +// " }\n" + +// lateinit var buildEnvs: List + + private val specialKey = listOf(".", "-") + + // private val specialValue = listOf("|", "&", "(", ")") + private val specialCharToReplace = Regex("['\n]") // --bug=75509999 Agent环境变量中替换掉破坏性字符 + private val logger = LoggerFactory.getLogger(BashUtil::class.java) + + fun execute( + buildId: String, + script: String, + dir: File, + buildEnvs: List, + runtimeVariables: Map, + continueNoneZero: Boolean = false, + systemEnvVariables: Map? = null, + prefix: String = "", + errorMessage: String? = null, + workspace: File = dir, + print2Logger: Boolean = true, + stepId: String? = null, + paramClassName: List + ): String { + return executeUnixCommand( + command = getCommandFile( + buildId = buildId, + script = script, + dir = dir, + workspace = workspace, + buildEnvs = buildEnvs, + runtimeVariables = runtimeVariables, + continueNoneZero = continueNoneZero, + systemEnvVariables = systemEnvVariables, + paramClassName = paramClassName + ).canonicalPath, + sourceDir = dir, + prefix = prefix, + errorMessage = errorMessage, + print2Logger = print2Logger, + executeErrorMessage = "", + buildId = buildId, + stepId = stepId + ) + } + + fun getCommandFile( + buildId: String, + script: String, + dir: File, + buildEnvs: List, + runtimeVariables: Map, + continueNoneZero: Boolean = false, + systemEnvVariables: Map? = null, + workspace: File = dir, + charSetType: CharsetType = CharsetType.UTF_8, + paramClassName: List + ): File { + val file = Files.createTempFile("devops_script", ".sh").toFile() + file.deleteOnExit() + + val command = StringBuilder() + val bashStr = script.split("\n")[0] + if (bashStr.startsWith("#!/")) { + command.append(bashStr).append("\n") + } + + command.append("export WORKSPACE=${workspace.absolutePath}\n") + .append("export DEVOPS_BUILD_SCRIPT_FILE=${file.absolutePath}\n") + + // 设置系统环境变量 + systemEnvVariables?.forEach { (name, value) -> + command.append("export $name=$value\n") + } + + val commonEnv = runtimeVariables + .filterNot { specialEnv(it.key) || it.key in paramClassName } + if (commonEnv.isNotEmpty()) { + commonEnv.forEach { (name, value) -> + // --bug=75509999 Agent环境变量中替换掉破坏性字符 + val clean = value.replace(specialCharToReplace, "") + command.append("export $name='$clean'\n") + } + } + if (buildEnvs.isNotEmpty()) { + var path = "" + buildEnvs.forEach { buildEnv -> + val home = File(getEnvironmentPathPrefix(), "${buildEnv.name}/${buildEnv.version}/") + if (!home.exists()) { + logger.error("环境变量路径(${home.absolutePath})不存在") + } + val envFile = File(home, buildEnv.binPath) + if (!envFile.exists()) { + logger.error("环境变量路径(${envFile.absolutePath})不存在") + return@forEach + } + // command.append("export $name=$path") + path = if (path.isEmpty()) { + envFile.absolutePath + } else { + "$path:${envFile.absolutePath}" + } + if (buildEnv.env.isNotEmpty()) { + buildEnv.env.forEach { (name, path) -> + val p = File(home, path) + command.append("export $name=${p.absolutePath}\n") + } + } + } + if (path.isNotEmpty()) { + path = "$path:\$PATH" + command.append("export PATH=$path\n") + } + } + + if (!continueNoneZero) { + command.append("set -e\n") + } else { + logger.info("每行命令运行返回值非零时，继续执行脚本") + command.append("set +e\n") + } + + command.append( + setEnv.replace( + oldValue = "##resultFile##", + newValue = File(dir, ScriptEnvUtils.getEnvFile(buildId)).absolutePath + ) + ) + + command.append( + format_multiple_lines.replace( + oldValue = "##resultFile##", + newValue = File(dir, ScriptEnvUtils.getMultipleLineFile(buildId)).absolutePath + ) + ) +// command.append( +// setGateValue.replace(oldValue = "##gateValueFile##", +// newValue = File(dir, ScriptEnvUtils.getQualityGatewayEnvFile()).absolutePath)) + command.append(script) + + val charset = when (charSetType) { + CharsetType.UTF_8 -> Charsets.UTF_8 + CharsetType.GBK -> Charset.forName(CharsetType.GBK.name) + else -> Charset.defaultCharset() + } + logger.info("The default charset is $charset") + + file.writeText(command.toString(), charset) + + if (AgentEnv.getOS() != OSType.WINDOWS) { + executeUnixCommand( + command = "chmod +x ${file.absolutePath}", + sourceDir = dir, + buildId = buildId + ) + } + CommonUtil.printTempFileInfo(file) + return file + } + + private fun executeUnixCommand( + command: String, + sourceDir: File, + prefix: String = "", + errorMessage: String? = null, + print2Logger: Boolean = true, + executeErrorMessage: String? = null, + buildId: String, + stepId: String? = null + ): String { + try { + return CommandLineUtils.execute( + command = command, + workspace = sourceDir, + print2Logger = print2Logger, + prefix = prefix, + executeErrorMessage = executeErrorMessage, + buildId = buildId, + stepId = stepId + ) + } catch (taskError: AtomException) { + throw taskError + } catch (ignored: Throwable) { + val errorInfo = errorMessage ?: "Fail to run the command $command" + logger.info("$errorInfo because of error(${ignored.message})") + throw AtomException( + ignored.message ?: "" + ) + } + } + + /*过滤处理特殊的key*/ + private fun specialEnv(key: String): Boolean { + return specialKey.any { key.contains(it) } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/BatScriptUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/BatScriptUtil.kt new file mode 100644 index 0000000..1a0ac4b --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/BatScriptUtil.kt @@ -0,0 +1,203 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils.script + +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.utils.CommandLineUtils +import com.tencent.bk.devops.atom.utils.CommonUtil +import com.tencent.bk.devops.atom.utils.ScriptEnvUtils +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.charset.Charset + +object BatScriptUtil { + + // + private const val setEnv = ":setEnv\r\n" + + " set file_save_dir=\"##resultFile##\"\r\n" + + " echo %~1=%~2 >>%file_save_dir%\r\n" + + " set %~1=%~2\r\n" + + " goto:eof\r\n" + + // + private const val setGateValue = ":setGateValue\r\n" + + " set file_save_dir=\"##gateValueFile##\"\r\n" + + " echo %~1=%~2 >>%file_save_dir%\r\n" + + " set %~1=%~2\r\n" + + " goto:eof\r\n" + + private val logger = LoggerFactory.getLogger(BatScriptUtil::class.java) + + // 2021-06-11 batchScript需要过滤掉上下文产生的变量，防止注入到环境变量中 + private val specialKey = listOf("variables.", "settings.", "envs.", "ci.", "job.", "jobs.", "steps.") + + private val specialValue = listOf("\n", "\r") + private val escapeValue = mapOf( + "&" to "^&", + "<" to "^<", + ">" to "^>", + "|" to "^|", + "\"" to "\\\"" + ) + + @Suppress("ALL") + fun execute( + script: String, + buildId: String, + runtimeVariables: Map, + dir: File, + prefix: String = "", + paramClassName: List, + errorMessage: String? = null, + workspace: File = dir, + print2Logger: Boolean = true, + stepId: String? = null, + charsetType: CharsetType? = null + ): String { + try { + val file = getCommandFile( + buildId = buildId, + script = script, + runtimeVariables = runtimeVariables, + dir = dir, + workspace = workspace, + charsetType = charsetType, + paramClassName = paramClassName + ) + return CommandLineUtils.execute( + command = "cmd.exe /C \"${file.canonicalPath}\"", + workspace = dir, + print2Logger = print2Logger, + prefix = prefix, + executeErrorMessage = "", + buildId = buildId, + stepId = stepId, + charSetType = charsetType + ) + } catch (ignore: Throwable) { + val errorInfo = errorMessage ?: "Fail to execute bat script" + logger.warn(errorInfo, ignore) + throw ignore + } + } + + @Suppress("ALL") + fun getCommandFile( + buildId: String, + script: String, + runtimeVariables: Map, + dir: File, + workspace: File = dir, + paramClassName: List, + charsetType: CharsetType? = null + ): File { + val tmpDir = System.getProperty("java.io.tmpdir") + val file = if (tmpDir.isNullOrBlank()) { + File.createTempFile("paas_build_script_", ".bat") + } else { + File(tmpDir).mkdirs() + File.createTempFile("paas_build_script_", ".bat", File(tmpDir)) + } + file.deleteOnExit() + + val command = StringBuilder() + + command.append("@echo off") + .append("\r\n") + .append("set WORKSPACE=${workspace.absolutePath}\r\n") + .append("set DEVOPS_BUILD_SCRIPT_FILE=${file.absolutePath}\r\n") + .append("\r\n") + + runtimeVariables +// .plus(CommonEnv.getCommonEnv()) // + .filterNot { specialEnv(it.key, it.value) || it.key in paramClassName } + .forEach { (name, value) -> + // 特殊保留字符转义 + val clean = escapeEnv(value) + command.append("set $name=\"$clean\"\r\n") // 双引号防止变量值有空格而意外截断定义 + command.append("set $name=%$name:~1,-1%\r\n") // 去除双引号，防止被程序读到有双引号的变量值 + } + + command.append(script.replace("\n", "\r\n")) + .append("\r\n") + .append("exit") + .append("\r\n") + .append( + setEnv.replace( + oldValue = "##resultFile##", + newValue = File(dir, ScriptEnvUtils.getEnvFile(buildId)).absolutePath + ) + ) + .append( + setGateValue.replace( + oldValue = "##gateValueFile##", + newValue = File(dir, ScriptEnvUtils.getQualityGatewayEnvFile()).canonicalPath + ) + ) + + val charset = when (charsetType) { + CharsetType.UTF_8 -> Charsets.UTF_8 + CharsetType.GBK -> Charset.forName(CharsetType.GBK.name) + else -> Charset.defaultCharset() + } + logger.info("The default charset is $charset") + + file.writeText(command.toString(), charset) + CommonUtil.printTempFileInfo(file) + return file + } + + private fun specialEnv(key: String, value: String): Boolean { + var match = false + /*过滤处理特殊的key*/ + for (it in specialKey) { + if (key.trim().startsWith(it)) { + match = true + break + } + } + + /*过滤处理特殊的value*/ + for (it in specialValue) { + if (value.contains(it)) { + match = true + break + } + } + return match + } + + /*做好转义，避免意外*/ + private fun escapeEnv(value: String): String { + var result = value + escapeValue.forEach { (k, v) -> + result = result.replace(k, v) + } + return result + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PowerShellUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PowerShellUtil.kt new file mode 100644 index 0000000..0bebaca --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PowerShellUtil.kt @@ -0,0 +1,162 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils.script + +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.utils.CommandLineUtils +import com.tencent.bk.devops.atom.utils.CommonUtil +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.charset.Charset + +object PowerShellUtil { + + // + private const val setEnv = "function setEnv(\$key, \$value)\n" + + "{\n" + + " Set-Item -Path Env:\\\$key -Value \$value\n" + + " echo \"::set-output name=\$key::\$value\"\n" + + "}\n" + + private val logger = LoggerFactory.getLogger(PowerShellUtil::class.java) + + // 2021-06-11 batchScript需要过滤掉上下文产生的变量，防止注入到环境变量中 + private val specialKey = listOf("variables.", "settings.", "envs.", "ci.", "job.", "jobs.", "steps.") + + private val specialValue = listOf("\n", "\r") + + @Suppress("ALL") + fun execute( + script: String, + buildId: String, + runtimeVariables: Map, + dir: File, + prefix: String = "", + paramClassName: List, + errorMessage: String? = null, + workspace: File = dir, + print2Logger: Boolean = true, + stepId: String? = null, + charsetType: CharsetType? = null + ): String { + try { + val file = getCommandFile( + buildId = buildId, + script = script, + runtimeVariables = runtimeVariables, + dir = dir, + workspace = workspace, + charsetType = charsetType, + paramClassName = paramClassName + ) + return CommandLineUtils.execute( + command = "powershell.exe /C \"${file.canonicalPath}\"", + workspace = dir, + print2Logger = print2Logger, + prefix = prefix, + executeErrorMessage = "", + buildId = buildId, + stepId = stepId, + charSetType = charsetType + ) + } catch (ignore: Throwable) { + val errorInfo = errorMessage ?: "Fail to execute bat script" + logger.warn(errorInfo, ignore) + throw ignore + } + } + + @Suppress("ALL") + fun getCommandFile( + buildId: String, + script: String, + runtimeVariables: Map, + dir: File, + workspace: File = dir, + paramClassName: List, + charsetType: CharsetType? = null + ): File { + val tmpDir = System.getProperty("java.io.tmpdir") + val file = if (tmpDir.isNullOrBlank()) { + File.createTempFile("devops_script", ".ps1") + } else { + File(tmpDir).mkdirs() + File.createTempFile("devops_script", ".ps1", File(tmpDir)) + } + file.deleteOnExit() + + val command = StringBuilder() + + command.append("Set-Item -Path Env:\\WORKSPACE -Value '${workspace.absolutePath}'\n") + .append("Set-Item -Path Env:\\DEVOPS_BUILD_SCRIPT_FILE -Value '${file.absolutePath}'\n") + .append("\r\n") + + runtimeVariables +// .plus(CommonEnv.getCommonEnv()) // + .filterNot { specialEnv(it.key, it.value) || it.key in paramClassName } + .forEach { (name, value) -> + command.append("Set-Item -Path Env:\\$name -Value '$value'\n") + } + + command.append(setEnv) + .append(script.replace("\n", "\r\n")) + .append("\r\n") + .append("exit") + + val charset = when (charsetType) { + CharsetType.UTF_8 -> Charsets.UTF_8 + CharsetType.GBK -> Charset.forName(CharsetType.GBK.name) + else -> Charset.defaultCharset() + } + logger.info("The default charset is $charset") + + file.writeText(command.toString(), charset) + CommonUtil.printTempFileInfo(file) + return file + } + + private fun specialEnv(key: String, value: String): Boolean { + var match = false + /*过滤处理特殊的key*/ + for (it in specialKey) { + if (key.trim().startsWith(it)) { + match = true + break + } + } + + /*过滤处理特殊的value*/ + for (it in specialValue) { + if (value.contains(it)) { + match = true + break + } + } + return match + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PwshUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PwshUtil.kt new file mode 100644 index 0000000..e1f9dc2 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PwshUtil.kt @@ -0,0 +1,175 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils.script + +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.utils.CommandLineUtils +import com.tencent.bk.devops.atom.utils.CommonUtil +import com.tencent.bk.devops.atom.utils.ScriptEnvUtils +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.charset.Charset +import java.nio.file.Files + +object PwshUtil { + + // + private const val setEnv = "function setEnv(\$key, \$value)\n" + + "{\n" + + " Set-Item -Path Env:\\\$key -Value \$value\n" + + " \"\$key=\$value\" | Out-File -Append ##resultFile##\n" + + "}\n" + + // + private const val setGateValue = "function setGateValue(\$key, \$value)\n" + + "{\n" + + " \"\$key=\$value\" | Out-File -Append ##gateValueFile##\n" + + "}\n" + + private val logger = LoggerFactory.getLogger(PwshUtil::class.java) + + // 2021-06-11 batchScript需要过滤掉上下文产生的变量，防止注入到环境变量中 + private val specialKey = listOf("variables.", "settings.", "envs.", "ci.", "job.", "jobs.", "steps.") + + private val specialValue = listOf("\n", "\r") + + @Suppress("ALL") + fun execute( + script: String, + buildId: String, + runtimeVariables: Map, + dir: File, + prefix: String = "", + paramClassName: List, + errorMessage: String? = null, + workspace: File = dir, + print2Logger: Boolean = true, + stepId: String? = null, + charsetType: CharsetType? = null + ): String { + try { + val file = getCommandFile( + buildId = buildId, + script = script, + runtimeVariables = runtimeVariables, + dir = dir, + workspace = workspace, + charsetType = charsetType, + paramClassName = paramClassName + ) + return CommandLineUtils.execute( + command = "pwsh \"${file.canonicalPath}\"", + workspace = dir, + print2Logger = print2Logger, + prefix = prefix, + executeErrorMessage = "", + buildId = buildId, + stepId = stepId, + charSetType = charsetType + ) + } catch (ignore: Throwable) { + val errorInfo = errorMessage ?: "Fail to execute bat script" + logger.warn(errorInfo, ignore) + throw ignore + } + } + + @Suppress("ALL") + fun getCommandFile( + buildId: String, + script: String, + runtimeVariables: Map, + dir: File, + workspace: File = dir, + paramClassName: List, + charsetType: CharsetType? = null + ): File { + val file = Files.createTempFile("devops_script", ".ps1").toFile() + file.deleteOnExit() + + val command = StringBuilder() + + command.append("Set-Item -Path Env:\\WORKSPACE -Value '${workspace.absolutePath}'\n") + .append("Set-Item -Path Env:\\DEVOPS_BUILD_SCRIPT_FILE -Value '${file.absolutePath}'\n") + .append("\r\n") + + runtimeVariables +// .plus(CommonEnv.getCommonEnv()) // + .filterNot { specialEnv(it.key, it.value) || it.key in paramClassName } + .forEach { (name, value) -> + command.append("Set-Item -Path Env:\\$name -Value '$value'\n") + } + + command.append( + setEnv.replace( + oldValue = "##resultFile##", + newValue = File(dir, ScriptEnvUtils.getEnvFile(buildId)).absolutePath + ) + ) + .append( + setGateValue.replace( + oldValue = "##gateValueFile##", + newValue = File(dir, ScriptEnvUtils.getQualityGatewayEnvFile()).canonicalPath + ) + ) + .append(script.replace("\n", "\r\n")) + .append("\r\n") + .append("exit") + + val charset = when (charsetType) { + CharsetType.UTF_8 -> Charsets.UTF_8 + CharsetType.GBK -> Charset.forName(CharsetType.GBK.name) + else -> Charset.defaultCharset() + } + logger.info("The default charset is $charset") + + file.writeText(command.toString(), charset) + CommonUtil.printTempFileInfo(file) + return file + } + + private fun specialEnv(key: String, value: String): Boolean { + var match = false + /*过滤处理特殊的key*/ + for (it in specialKey) { + if (key.trim().startsWith(it)) { + match = true + break + } + } + + /*过滤处理特殊的value*/ + for (it in specialValue) { + if (value.contains(it)) { + match = true + break + } + } + return match + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PythonUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PythonUtil.kt new file mode 100644 index 0000000..59f3e91 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/PythonUtil.kt @@ -0,0 +1,265 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils.script + +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.enums.OSType +import com.tencent.bk.devops.atom.exception.AtomException +import com.tencent.bk.devops.atom.pojo.AgentEnv +import com.tencent.bk.devops.atom.pojo.BuildEnv +import com.tencent.bk.devops.atom.utils.CommandLineUtils +import com.tencent.bk.devops.atom.utils.CommonUtil +import com.tencent.bk.devops.atom.utils.ScriptEnvUtils +import com.tencent.bk.devops.atom.utils.getEnvironmentPathPrefix +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.charset.Charset +import java.nio.file.Files + +@Suppress("ALL") +object PythonUtil { + + // + private const val setEnv = "def setEnv(key,value):\n" + + " os.environ[key]=value\n" + + " f = open(\"##resultFile##\", 'a+')\n" + + " print(\"{0}={1}\".format(key, value), file=f)\n" + private const val format_multiple_lines = "def format_multiple_lines(s: str):\n" + + " out = s.replace('%','%25').replace('\\n','%0A').replace('\\r','%0D')\n" + + " f = open(\"##resultFile##\", 'a+')\n" + + " print(out, file=f)\n" + + " return out\n" + // +// private const val setGateValue = "setGateValue(){\n" + +// " local key=\$1\n" + +// " local val=\$2\n" + +// "\n" + +// " if [[ -z \"\$@\" ]]; then\n" + +// " return 0\n" + +// " fi\n" + +// "\n" + +// " if ! echo \"\$key\" | grep -qE \"^[a-zA-Z_][a-zA-Z0-9_]*\$\"; then\n" + +// " echo \"[\$key] is invalid\" >&2\n" + +// " return 1\n" + +// " fi\n" + +// "\n" + +// " echo \$key=\$val >> ##gateValueFile##\n" + +// " }\n" + +// lateinit var buildEnvs: List + + private val specialKey = listOf(".", "-") + + // private val specialValue = listOf("|", "&", "(", ")") + private val specialCharToReplace = Regex("['\n]") // --bug=75509999 Agent环境变量中替换掉破坏性字符 + private val logger = LoggerFactory.getLogger(PythonUtil::class.java) + + fun execute( + buildId: String, + script: String, + dir: File, + buildEnvs: List, + runtimeVariables: Map, + continueNoneZero: Boolean = false, + systemEnvVariables: Map? = null, + prefix: String = "", + errorMessage: String? = null, + workspace: File = dir, + print2Logger: Boolean = true, + stepId: String? = null, + paramClassName: List, + charsetType: CharsetType? = null + ): String { + return executeUnixCommand( + command = "python3 " + getCommandFile( + buildId = buildId, + script = script, + dir = dir, + workspace = workspace, + buildEnvs = buildEnvs, + runtimeVariables = runtimeVariables, + continueNoneZero = continueNoneZero, + systemEnvVariables = systemEnvVariables, + paramClassName = paramClassName, + charsetType = charsetType + ).canonicalPath, + sourceDir = dir, + prefix = prefix, + errorMessage = errorMessage, + print2Logger = print2Logger, + executeErrorMessage = "", + buildId = buildId, + stepId = stepId, + charsetType = charsetType + ) + } + + fun getCommandFile( + buildId: String, + script: String, + dir: File, + buildEnvs: List, + runtimeVariables: Map, + continueNoneZero: Boolean = false, + systemEnvVariables: Map? = null, + workspace: File = dir, + charSetType: CharsetType = CharsetType.UTF_8, + paramClassName: List, + charsetType: CharsetType? = null + ): File { + val file = Files.createTempFile("devops_script", ".py").toFile() + file.deleteOnExit() + + val command = StringBuilder() + + command.append("import os\n") + .append("os.environ['WORKSPACE']='${workspace.absolutePath}'\n") + .append("os.environ['DEVOPS_BUILD_SCRIPT_FILE']='${file.absolutePath}'\n") + + // 设置系统环境变量 + systemEnvVariables?.forEach { (name, value) -> + command.append("os.environ['$name']='$value'\n") + } + + val commonEnv = runtimeVariables + .filterNot { specialEnv(it.key) || it.key in paramClassName } + if (commonEnv.isNotEmpty()) { + commonEnv.forEach { (name, value) -> + // --bug=75509999 Agent环境变量中替换掉破坏性字符 + val clean = value.replace(specialCharToReplace, "") + command.append("os.environ['$name']='$clean'\n") + } + } + if (buildEnvs.isNotEmpty()) { + var path = "" + buildEnvs.forEach { buildEnv -> + val home = File(getEnvironmentPathPrefix(), "${buildEnv.name}/${buildEnv.version}/") + if (!home.exists()) { + logger.error("环境变量路径(${home.absolutePath})不存在") + } + val envFile = File(home, buildEnv.binPath) + if (!envFile.exists()) { + logger.error("环境变量路径(${envFile.absolutePath})不存在") + return@forEach + } + // command.append("export $name=$path") + path = if (path.isEmpty()) { + envFile.absolutePath + } else { + "$path:${envFile.absolutePath}" + } + if (buildEnv.env.isNotEmpty()) { + buildEnv.env.forEach { (name, path) -> + val p = File(home, path) + command.append("$name='${p.absolutePath}'\n") + } + } + } + if (path.isNotEmpty()) { + path = "$path:\$PATH" + command.append("os.environ['PATH']='$path'\n") + } + } + + command.append( + setEnv.replace( + oldValue = "##resultFile##", + newValue = File(dir, ScriptEnvUtils.getEnvFile(buildId)).absolutePath.replace("\\", "/") + ) + ) + + command.append( + format_multiple_lines.replace( + oldValue = "##resultFile##", + newValue = File(dir, ScriptEnvUtils.getMultipleLineFile(buildId)).absolutePath + ) + ) +// command.append( +// setGateValue.replace(oldValue = "##gateValueFile##", +// newValue = File(dir, ScriptEnvUtils.getQualityGatewayEnvFile()).absolutePath)) + command.append(script) + + val charset = when (charSetType) { + CharsetType.UTF_8 -> Charsets.UTF_8 + CharsetType.GBK -> Charset.forName(CharsetType.GBK.name) + else -> Charset.defaultCharset() + } + logger.info("The default charset is $charset") + + file.writeText(command.toString(), charset) + + if (AgentEnv.getOS() != OSType.WINDOWS) { + executeUnixCommand( + command = "chmod +x ${file.absolutePath}", + sourceDir = dir, + buildId = buildId, + charsetType = charsetType + ) + } + CommonUtil.printTempFileInfo(file) + return file + } + + private fun executeUnixCommand( + command: String, + sourceDir: File, + prefix: String = "", + errorMessage: String? = null, + print2Logger: Boolean = true, + executeErrorMessage: String? = null, + buildId: String, + stepId: String? = null, + charsetType: CharsetType? = null + ): String { + try { + return CommandLineUtils.execute( + command = command, + workspace = sourceDir, + print2Logger = print2Logger, + prefix = prefix, + executeErrorMessage = executeErrorMessage, + buildId = buildId, + charSetType = charsetType, + stepId = stepId + ) + } catch (taskError: AtomException) { + throw taskError + } catch (ignored: Throwable) { + val errorInfo = errorMessage ?: "Fail to run the command $command" + logger.info("$errorInfo because of error(${ignored.message})") + throw AtomException( + ignored.message ?: "" + ) + } + } + + /*过滤处理特殊的key*/ + private fun specialEnv(key: String): Boolean { + return specialKey.any { key.contains(it) } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/ShUtil.kt b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/ShUtil.kt new file mode 100644 index 0000000..7e2a07b --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/atom/utils/script/ShUtil.kt @@ -0,0 +1,273 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bk.devops.atom.utils.script + +import com.tencent.bk.devops.atom.enums.CharsetType +import com.tencent.bk.devops.atom.exception.AtomException +import com.tencent.bk.devops.atom.pojo.BuildEnv +import com.tencent.bk.devops.atom.utils.CommandLineUtils +import com.tencent.bk.devops.atom.utils.CommonUtil +import com.tencent.bk.devops.atom.utils.ScriptEnvUtils +import com.tencent.bk.devops.atom.utils.getEnvironmentPathPrefix +import org.slf4j.LoggerFactory +import java.io.File +import java.nio.charset.Charset +import java.nio.file.Files + +@Suppress("ALL") +object ShUtil { + + // + private const val setEnv = "setEnv(){\n" + + " local key=\$1\n" + + " local val=\$2\n" + + "\n" + + " if [[ -z \"\$@\" ]]; then\n" + + " return 0\n" + + " fi\n" + + "\n" + + " if ! echo \"\$key\" | grep -qE \"^[a-zA-Z_][a-zA-Z0-9_]*\$\"; then\n" + + " echo \"[\$key] is invalid\" >&2\n" + + " return 1\n" + + " fi\n" + + "\n" + + " echo \$key=\$val >> ##resultFile##\n" + + " export \$key=\"\$val\"\n" + + " }\n" + private const val format_multiple_lines = "format_multiple_lines() {\n" + + " local content=\$1\n" + + " content=\"\${content//'%'/'%25'}\"\n" + + " content=\"\${content//\$'\\n'/'%0A'}\"\n" + + " content=\"\${content//\$'\\r'/'%0D'}\"\n" + + " echo \"\$content\"\n" + + "}\n" + // +// private const val setGateValue = "setGateValue(){\n" + +// " local key=\$1\n" + +// " local val=\$2\n" + +// "\n" + +// " if [[ -z \"\$@\" ]]; then\n" + +// " return 0\n" + +// " fi\n" + +// "\n" + +// " if ! echo \"\$key\" | grep -qE \"^[a-zA-Z_][a-zA-Z0-9_]*\$\"; then\n" + +// " echo \"[\$key] is invalid\" >&2\n" + +// " return 1\n" + +// " fi\n" + +// "\n" + +// " echo \$key=\$val >> ##gateValueFile##\n" + +// " }\n" + +// lateinit var buildEnvs: List + + private val specialKey = listOf(".", "-") + + // private val specialValue = listOf("|", "&", "(", ")") + private val specialCharToReplace = Regex("['\n]") // --bug=75509999 Agent环境变量中替换掉破坏性字符 + private val logger = LoggerFactory.getLogger(ShUtil::class.java) + + fun execute( + buildId: String, + script: String, + dir: File, + buildEnvs: List, + runtimeVariables: Map, + continueNoneZero: Boolean = false, + systemEnvVariables: Map? = null, + prefix: String = "", + errorMessage: String? = null, + workspace: File = dir, + print2Logger: Boolean = true, + stepId: String? = null, + paramClassName: List + ): String { + return executeUnixCommand( + command = "sh -e " + getCommandFile( + buildId = buildId, + script = script, + dir = dir, + workspace = workspace, + buildEnvs = buildEnvs, + runtimeVariables = runtimeVariables, + continueNoneZero = continueNoneZero, + systemEnvVariables = systemEnvVariables, + paramClassName = paramClassName + ).canonicalPath, + sourceDir = dir, + prefix = prefix, + errorMessage = errorMessage, + print2Logger = print2Logger, + executeErrorMessage = "", + buildId = buildId, + stepId = stepId + ) + } + + fun getCommandFile( + buildId: String, + script: String, + dir: File, + buildEnvs: List, + runtimeVariables: Map, + continueNoneZero: Boolean = false, + systemEnvVariables: Map? = null, + workspace: File = dir, + charSetType: CharsetType = CharsetType.UTF_8, + paramClassName: List + ): File { + val file = Files.createTempFile("devops_script", ".sh").toFile() + file.deleteOnExit() + + val command = StringBuilder() + val bashStr = script.split("\n")[0] + if (bashStr.startsWith("#!/")) { + command.append(bashStr).append("\n") + } + + command.append("export WORKSPACE=${workspace.absolutePath}\n") + .append("export DEVOPS_BUILD_SCRIPT_FILE=${file.absolutePath}\n") + + // 设置系统环境变量 + systemEnvVariables?.forEach { (name, value) -> + command.append("export $name=$value\n") + } + + val commonEnv = runtimeVariables + .filterNot { specialEnv(it.key) || it.key in paramClassName } + if (commonEnv.isNotEmpty()) { + commonEnv.forEach { (name, value) -> + // --bug=75509999 Agent环境变量中替换掉破坏性字符 + val clean = value.replace(specialCharToReplace, "") + command.append("export $name='$clean'\n") + } + } + if (buildEnvs.isNotEmpty()) { + var path = "" + buildEnvs.forEach { buildEnv -> + val home = File(getEnvironmentPathPrefix(), "${buildEnv.name}/${buildEnv.version}/") + if (!home.exists()) { + logger.error("环境变量路径(${home.absolutePath})不存在") + } + val envFile = File(home, buildEnv.binPath) + if (!envFile.exists()) { + logger.error("环境变量路径(${envFile.absolutePath})不存在") + return@forEach + } + // command.append("export $name=$path") + path = if (path.isEmpty()) { + envFile.absolutePath + } else { + "$path:${envFile.absolutePath}" + } + if (buildEnv.env.isNotEmpty()) { + buildEnv.env.forEach { (name, path) -> + val p = File(home, path) + command.append("export $name=${p.absolutePath}\n") + } + } + } + if (path.isNotEmpty()) { + path = "$path:\$PATH" + command.append("export PATH=$path\n") + } + } + + if (!continueNoneZero) { + command.append("set -e\n") + } else { + logger.info("每行命令运行返回值非零时，继续执行脚本") + command.append("set +e\n") + } + + command.append( + setEnv.replace( + oldValue = "##resultFile##", + newValue = File(dir, ScriptEnvUtils.getEnvFile(buildId)).absolutePath + ) + ) + + command.append(format_multiple_lines) +// command.append( +// setGateValue.replace(oldValue = "##gateValueFile##", +// newValue = File(dir, ScriptEnvUtils.getQualityGatewayEnvFile()).absolutePath)) + command.append(script) + + val charset = when (charSetType) { + CharsetType.UTF_8 -> Charsets.UTF_8 + CharsetType.GBK -> Charset.forName(CharsetType.GBK.name) + else -> Charset.defaultCharset() + } + logger.info("The default charset is $charset") + + file.writeText(command.toString(), charset) + + executeUnixCommand( + command = "chmod +x ${file.absolutePath}", + sourceDir = dir, + buildId = buildId + ) + CommonUtil.printTempFileInfo(file) + return file + } + + private fun executeUnixCommand( + command: String, + sourceDir: File, + prefix: String = "", + errorMessage: String? = null, + print2Logger: Boolean = true, + executeErrorMessage: String? = null, + buildId: String, + stepId: String? = null + ): String { + try { + return CommandLineUtils.execute( + command = command, + workspace = sourceDir, + print2Logger = print2Logger, + prefix = prefix, + executeErrorMessage = executeErrorMessage, + buildId = buildId, + stepId = stepId + ) + } catch (taskError: AtomException) { + throw taskError + } catch (ignored: Throwable) { + val errorInfo = errorMessage ?: "Fail to run the command $command" + logger.info("$errorInfo because of error(${ignored.message})") + throw AtomException( + ignored.message ?: "" + ) + } + } + + /*过滤处理特殊的key*/ + private fun specialEnv(key: String): Boolean { + return specialKey.any { key.contains(it) } + } +} diff --git a/src/main/resources/META-INF/services/com.tencent.bk.devops.atom.spi.TaskAtom b/src/main/resources/META-INF/services/com.tencent.bk.devops.atom.spi.TaskAtom new file mode 100644 index 0000000..6a467b1 --- /dev/null +++ b/src/main/resources/META-INF/services/com.tencent.bk.devops.atom.spi.TaskAtom @@ -0,0 +1 @@ +com.tencent.bk.devops.atom.ScriptRunAtom diff --git a/src/test/resources/.sdk.json b/src/test/resources/.sdk.json new file mode 100644 index 0000000..efaeacd --- /dev/null +++ b/src/test/resources/.sdk.json @@ -0,0 +1,9 @@ +{ + "buildType": "DOCKER" , + "projectId": "232", + "agentId": "1x", + "secretKey": "2323232", + "gateway": "api.github.com", + "buildId": "vvadsaaf", + "vmSeqId": "33223" +} diff --git a/src/test/resources/input.json b/src/test/resources/input.json new file mode 100644 index 0000000..5377971 --- /dev/null +++ b/src/test/resources/input.json @@ -0,0 +1,3 @@ +{ + "desc": "hello world!" +} diff --git a/task.json b/task.json new file mode 100644 index 0000000..12f22ab --- /dev/null +++ b/task.json @@ -0,0 +1,86 @@ +{ + "atomCode": "run", + "execution": { + "packagePath": "run-jar-with-dependencies.jar", + "language": "java", + "minimumVersion": "1.8", + "demands": [], + "target": "java -jar run-jar-with-dependencies.jar" + }, + "input": { + "shell": { + "rule": {}, + "type": "enum-input", + "label": "指定脚本语言", + "desc": "指定脚本语言，默认时Windows执行Batch，Linux和Macos执行Shell。", + "required": false, + "hidden": false, + "component": "enum-input", + "list": [ + { + "value": "auto", + "label": "默认" + },{ + "value": "bash", + "label": "BASH" + }, + { + "value": "cmd", + "label": "CMD" + }, + { + "value": "pwsh", + "label": "POWERSHELL_CORE" + }, + { + "value": "powershell", + "label": "POWERSHELL_DESKTOP" + }, + { + "value": "python", + "label": "PYTHON" + }, + { + "value": "sh", + "label": "SH" + } + ], + "default": "auto" + }, + "script": { + "label": "执行脚本", + "default": "", + "placeholder": "输入脚本", + "type": "atom-ace-editor", + "desc": "输入脚本", + "required": true, + "disabled": false, + "hidden": false, + "isSensitive": false + }, + "charsetType": { + "rule": {}, + "type": "enum-input", + "label": "windows下字符集类型", + "desc": "仅windows构建机所需参数", + "required": false, + "hidden": false, + "component": "enum-input", + "list": [ + { + "value": "DEFAULT", + "label": "DEFAULT" + }, + { + "value": "UTF_8", + "label": "UTF-8" + }, + { + "value": "GBK", + "label": "GBK" + } + ], + "default": "DEFAULT" + } + } +}