Dev tming

src/backend/booster/bk_dist/common/types/error.go

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2021 THL A29 Limited, a Tencent company. All rights reserved
+ *
+ * This source code file is licensed under the MIT License, you may obtain a copy of the License at
+ *
+ * http://opensource.org/licenses/MIT
+ *
+ */
+
+package types
+
+import "fmt"
+
+type BKDistCommonError struct {
+	Code  int
+	Error error
+}
+
+var (
+	DescPreForceLocal       = fmt.Errorf("cmd in force local when pre execute")
+	DescPreNotSupportRemote = fmt.Errorf("cmd not support remote when pre execute")
+
+	DescRemoteSendFile      = fmt.Errorf("send file failed when remote execute")
+	DescRemoteSendToolchain = fmt.Errorf("send toolchain failed when remote execute")
+
+	DescPostMissPumpDependFile   = fmt.Errorf("miss pump depend file when post execute")
+	DescPostMissNormalDependFile = fmt.Errorf("miss normal depend file when post execute")
+	DescPostSaveFileFailed       = fmt.Errorf("save file failed when post execute")
+	DescPostOutOfMemoy           = fmt.Errorf("out of memory when post execute")
+	DescPostToolchainNotFound    = fmt.Errorf("toolchain not found when post execute")
+
+	DescUnknown = fmt.Errorf("unknown")
+)
+
+// define errors
+var (
+	// no error        0
+	ErrorNone = BKDistCommonError{Code: 0, Error: nil}
+
+	// pre-execute     1~999
+	ErrorPreForceLocal       = BKDistCommonError{Code: 1, Error: DescPreForceLocal}
+	ErrorPreNotSupportRemote = BKDistCommonError{Code: 2, Error: DescPreNotSupportRemote}
+
+	// remote-execute  1000~1999
+	ErrorRemoteSendFile      = BKDistCommonError{Code: 1000, Error: DescRemoteSendFile}
+	ErrorRemoteSendToolchain = BKDistCommonError{Code: 1001, Error: DescRemoteSendToolchain}
+
+	// post-execute    2000~2999
+	ErrorPostMissPumpDependFile   = BKDistCommonError{Code: 2000, Error: DescPostMissPumpDependFile}
+	ErrorPostMissNormalDependFile = BKDistCommonError{Code: 2001, Error: DescPostMissNormalDependFile}
+	ErrorPostSaveFileFailed       = BKDistCommonError{Code: 2002, Error: DescPostSaveFileFailed}
+	ErrorPostOutOfMemoy           = BKDistCommonError{Code: 2100, Error: DescPostOutOfMemoy}
+	ErrorPostToolchainNotFound    = BKDistCommonError{Code: 2200, Error: DescPostToolchainNotFound}
+
+	// local-execute   3000~3999
+
+	// other error     999999
+	ErrorUnknown = BKDistCommonError{Code: 999999, Error: DescUnknown}
+)

src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go

@@ -155,7 +155,7 @@ func (e *executor) executePreTask() (*dcSDK.BKDistCommand, error) {
 
 	dcSDK.StatsTimeNow(&e.stats.PreWorkStartTime)
 	e.mgr.work.Basic().UpdateJobStats(e.stats)
-	r, err := e.handler.PreExecute(e.req.Commands)
+	r, _, err := e.handler.PreExecute(e.req.Commands)
 	dcSDK.StatsTimeNow(&e.stats.PreWorkEndTime)
 	if err != nil {
 		return nil, err
@@ -211,7 +211,7 @@ func (e *executor) onRemoteFail() (*dcSDK.BKDistCommand, error) {
 
 	// dcSDK.StatsTimeNow(&e.stats.PreWorkStartTime)
 	// e.mgr.work.Basic().UpdateJobStats(e.stats)
-	r, err := e.handler.OnRemoteFail(e.req.Commands)
+	r, _, err := e.handler.OnRemoteFail(e.req.Commands)
 	// dcSDK.StatsTimeNow(&e.stats.PreWorkEndTime)
 	if err != nil {
 		return nil, err
@@ -247,7 +247,7 @@ func (e *executor) executePostTask(result *dcSDK.BKDistResult) error {
 	dcSDK.StatsTimeNow(&e.stats.PostWorkStartTime)
 	defer dcSDK.StatsTimeNow(&e.stats.PostWorkEndTime)
 	e.mgr.work.Basic().UpdateJobStats(e.stats)
-	err := e.handler.PostExecute(result)
+	_, err := e.handler.PostExecute(result)
 	if err != nil {
 		return err
 	}

src/backend/booster/bk_dist/handler/cc/handler.go

@@ -27,6 +27,7 @@ import (
 	dcPump "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/pump"
 	dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk"
 	dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall"
+	dcType "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types"
 	dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/controller/pkg/manager/analyser"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler"
@@ -150,7 +151,7 @@ func (cc *TaskCC) PreLockWeight(command []string) int32 {
 }
 
 // PreExecute 预处理
-func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, error) {
+func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, int, error) {
 	return cc.preExecute(command)
 }
 
@@ -185,7 +186,7 @@ func (cc *TaskCC) NeedRetryOnRemoteFail(command []string) bool {
 }
 
 // OnRemoteFail give chance to try other way if failed to remote execute
-func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) {
+func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, int, error) {
 	blog.Infof("cc: start OnRemoteFail for: %v", command)
 
 	if cc.pumpremote {
@@ -195,7 +196,8 @@ func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) {
 		cc.pumpremote = false
 		return cc.preExecute(command)
 	}
-	return nil, nil
+
+	return nil, dcType.ErrorNone.Code, dcType.ErrorNone.Error
 }
 
 // PostLockWeight decide post-execute lock weight, default 1
@@ -204,7 +206,7 @@ func (cc *TaskCC) PostLockWeight(result *dcSDK.BKDistResult) int32 {
 }
 
 // PostExecute 后置处理, 判断远程执行的结果是否正确
-func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) error {
+func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) (int, error) {
 	return cc.postExecute(r)
 }
 
@@ -758,7 +760,7 @@ func (cc *TaskCC) workerSupportAbsPath() bool {
 	return true
 }
 
-func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
+func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, int, error) {
 	blog.Infof("cc: [%s] start pre execute for: %v", cc.tag, command)
 
 	cc.originArgs = command
@@ -770,25 +772,25 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 		if err != nil {
 			if notifyerr == ErrorNotSupportRemote {
 				blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err)
-				return nil, err
+				return nil, dcType.ErrorPreNotSupportRemote.Code, dcType.ErrorPreNotSupportRemote.Error
 			}
 		} else {
 			// for debug
 			blog.Debugf("cc: after try pump, req: %+v", *req)
 			cc.pumpremote = true
-			return req, err
+			return req, dcType.ErrorNone.Code, dcType.ErrorNone.Error
 		}
 	}
 
 	compilerEnsuredArgs, err := ensureCompiler(command)
 	if err != nil {
 		blog.Warnf("cc: [%s] pre execute ensure compiler %v: %v", cc.tag, command, err)
-		return nil, err
+		return nil, dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 	}
 	args, err := expandOptions(cc.sandbox, compilerEnsuredArgs)
 	if err != nil {
 		blog.Warnf("cc: [%s] pre execute expand options %v: %v", cc.tag, compilerEnsuredArgs, err)
-		return nil, err
+		return nil, dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 	}
 	cc.ensuredArgs = args
 
@@ -807,7 +809,8 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 		for _, v1 := range cc.ForceLocalCppFileKeys {
 			if v1 != "" && strings.Contains(v, v1) {
 				blog.Warnf("cc: pre execute found %s is in force local list, do not deal now", v)
-				return nil, fmt.Errorf("arg %s is in force local cpp list", v)
+				// return nil, fmt.Errorf("arg %s is in force local cpp list", v)
+				return nil, dcType.ErrorPreForceLocal.Code, dcType.ErrorPreForceLocal.Error
 			}
 		}
 	}
@@ -820,7 +823,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 
 	if err = cc.preBuild(args); err != nil {
 		blog.Warnf("cc: [%s] pre execute pre-build %v: %v", cc.tag, args, err)
-		return nil, err
+		return nil, dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 	}
 
 	// generate the input files for pre-process file
@@ -829,7 +832,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 		if !existed {
 			err := fmt.Errorf("result file %s not existed", cc.preprocessedFile)
 			blog.Warnf("cc: [%s] %v", cc.tag, err)
-			return nil, err
+			return nil, dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 		}
 
 		cc.sendFiles = append(cc.sendFiles, dcSDK.FileDesc{
@@ -870,13 +873,13 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 			},
 		},
 		CustomSave: true,
-	}, nil
+	}, dcType.ErrorNone.Code, dcType.ErrorNone.Error
 }
 
-func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error {
+func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) (int, error) {
 	blog.Infof("cc: [%s] start post execute", cc.tag)
 	if r == nil || len(r.Results) == 0 {
-		return ErrorInvalidParam
+		return dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 	}
 
 	resultfilenum := 0
@@ -893,7 +896,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error {
 			if f.Buffer != nil {
 				if err := saveResultFile(&f, cc.sandbox.Dir); err != nil {
 					blog.Errorf("cc: failed to save file [%s]", f.FilePath)
-					return err
+					return dcType.ErrorPostSaveFileFailed.Code, dcType.ErrorPostSaveFileFailed.Error
 				}
 				resultfilenum++
 			}
@@ -914,7 +917,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error {
 		if cc.pumpremote {
 			cc.needcopypumpheadfile = false
 		}
-		return nil
+		return dcType.ErrorNone.Code, dcType.ErrorNone.Error
 	}
 
 ERROREND:
@@ -939,11 +942,13 @@ ERROREND:
 		os.Remove(cc.pumpHeadFile)
 	}
 
-	return fmt.Errorf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s",
+	blog.Warnf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s",
 		cc.tag,
 		r.Results[0].RetCode,
 		r.Results[0].ErrorMessage,
 		r.Results[0].OutputMessage)
+
+	return dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 }
 
 func (cc *TaskCC) ensureOwner(fdl []string) {
@@ -1170,6 +1175,22 @@ func (cc *TaskCC) preBuild(args []string) error {
 				break
 			}
 		}
+
+		// avoid error : does not allow 'register' storage class specifier
+		prefix := "-std=c++"
+		for _, v := range serverSideArgs {
+			if strings.HasPrefix(v, prefix) {
+				if len(v) > len(prefix) {
+					version, err := strconv.Atoi(v[len(prefix):])
+					if err == nil && version > 14 {
+						serverSideArgs = append(serverSideArgs, "-Wno-register")
+						blog.Infof("cc: found %s,ready add [-Wno-register]", v)
+					}
+				}
+				break
+			}
+		}
+
 		cc.serverSideArgs = serverSideArgs
 
 		if cc.isPumpCheck() && pumpErr == nil {

src/backend/booster/bk_dist/handler/custom/handler.go

@@ -135,7 +135,7 @@ func (c *Custom) PreLockWeight(command []string) int32 {
 }
 
 // PreExecute 单个任务的预处理, 如c/c++编译的pre-process, 决定了分发到远程处理的任务信息
-func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, error) {
+func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, int, error) {
 	return c.innerHandler.PreExecute(command)
 }
 
@@ -175,8 +175,8 @@ func (c *Custom) NeedRetryOnRemoteFail(command []string) bool {
 }
 
 // OnRemoteFail give chance to try other way if failed to remote execute
-func (c *Custom) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) {
-	return nil, nil
+func (c *Custom) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, int, error) {
+	return nil, dcType.ErrorNone.Code, dcType.ErrorNone.Error
 }
 
 // PostLockWeight decide post-execute lock weight, default 1
@@ -185,7 +185,7 @@ func (c *Custom) PostLockWeight(result *dcSDK.BKDistResult) int32 {
 }
 
 // PostExecute 单个任务的后置处理, 需要处理远程任务执行的结果
-func (c *Custom) PostExecute(result *dcSDK.BKDistResult) error {
+func (c *Custom) PostExecute(result *dcSDK.BKDistResult) (int, error) {
 	return c.innerHandler.PostExecute(result)
 }
 

src/backend/booster/bk_dist/handler/echo/handler.go

@@ -10,7 +10,6 @@
 package echo
 
 import (
-	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -140,11 +139,12 @@ func (c *Echo) PreLockWeight(command []string) int32 {
 
 // PreExecute do the pre-process in one executor command.
 // PreExecute should analyse the input command and generate the DistCommand to send to remote.
-func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, error) {
+func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, int, error) {
 	upperCommand := strings.ToUpper(command[0])
 	if len(command) < 2 || (!strings.HasSuffix(upperCommand, "ECHO") &&
 		!strings.HasSuffix(upperCommand, "ECHO.EXE")) {
-		return nil, fmt.Errorf("invalid command,len(command):[%d], command[0]:[%s]", len(command), command[0])
+		blog.Warnf("echo: invalid command,len(command):[%d], command[0]:[%s]", len(command), command[0])
+		return nil, dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 	}
 
 	inputfiles := make([]dcSDK.FileDesc, 0, 1)
@@ -165,7 +165,7 @@ func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, error) {
 				existed, fileSize, modifyTime, fileMode = dcFile.Stat(newinput).Batch()
 				if !existed {
 					blog.Infof("echo: input file %s not exist", newinput)
-					return nil, fmt.Errorf("echo: input file %s not exist", command[i])
+					return nil, dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 				}
 			}
 		}
@@ -194,7 +194,7 @@ func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, error) {
 			},
 		},
 		CustomSave: true,
-	}, nil
+	}, dcType.ErrorNone.Code, dcType.ErrorNone.Error
 }
 
 // NeedRemoteResource check whether this command need remote resource
@@ -213,8 +213,8 @@ func (c *Echo) NeedRetryOnRemoteFail(command []string) bool {
 }
 
 // OnRemoteFail give chance to try other way if failed to remote execute
-func (c *Echo) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) {
-	return nil, nil
+func (c *Echo) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, int, error) {
+	return nil, dcType.ErrorNone.Code, dcType.ErrorNone.Error
 }
 
 // PostLockWeight decide post-execute lock weight, default 1
@@ -225,24 +225,27 @@ func (c *Echo) PostLockWeight(result *dcSDK.BKDistResult) int32 {
 // PostExecute do the post-process in one executor command.
 // PostExecute should check the DistResult and judge whether the remote processing succeeded.
 // Also PostExecute should manages the output message.
-func (c *Echo) PostExecute(r *dcSDK.BKDistResult) error {
+func (c *Echo) PostExecute(r *dcSDK.BKDistResult) (int, error) {
 	// do not save result to disk
 	if r == nil || len(r.Results) == 0 {
-		return fmt.Errorf("echo: result data is invalid")
+		blog.Warnf("echo: result data is invalid")
+		return dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 	}
 
 	result := r.Results[0]
 	if len(result.ResultFiles) == 0 {
-		return fmt.Errorf("echo: not found result file, retcode %d, error message:[%s], output message:[%s]",
+		blog.Warnf("echo: not found result file, retcode %d, error message:[%s], output message:[%s]",
 			result.RetCode,
 			result.ErrorMessage,
 			result.OutputMessage)
+
+		return dcType.ErrorUnknown.Code, dcType.ErrorUnknown.Error
 	}
 
 	for _, v := range result.ResultFiles {
 		blog.Infof("echo: received result file[%s],len[%d]", v.FilePath, len(v.Buffer))
 	}
-	return nil
+	return dcType.ErrorNone.Code, dcType.ErrorNone.Error
 }
 
 // FinalExecute provide a chance to do process before the process exit.