Dev tming

src/backend/booster/bk_dist/common/types/error.go

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2021 THL A29 Limited, a Tencent company. All rights reserved
+ *
+ * This source code file is licensed under the MIT License, you may obtain a copy of the License at
+ *
+ * http://opensource.org/licenses/MIT
+ *
+ */
+
+package types
+
+import "fmt"
+
+type BKDistCommonError struct {
+	Code  int
+	Error error
+}
+
+var (
+	DescPreForceLocal       = fmt.Errorf("cmd in force local when pre execute")
+	DescPreNotSupportRemote = fmt.Errorf("cmd not support remote when pre execute")
+
+	DescRemoteSendFile      = fmt.Errorf("send file failed when remote execute")
+	DescRemoteSendToolchain = fmt.Errorf("send toolchain failed when remote execute")
+
+	DescPostMissPumpDependFile   = fmt.Errorf("miss pump depend file when post execute")
+	DescPostMissNormalDependFile = fmt.Errorf("miss normal depend file when post execute")
+	DescPostSaveFileFailed       = fmt.Errorf("save file failed when post execute")
+	DescPostOutOfMemoy           = fmt.Errorf("out of memory when post execute")
+	DescPostToolchainNotFound    = fmt.Errorf("toolchain not found when post execute")
+
+	DescUnknown = fmt.Errorf("unknown")
+)
+
+const (
+	BaseCode = 100000
+)
+
+// define errors
+var (
+	// no error        0
+	ErrorNone = BKDistCommonError{Code: 0, Error: nil}
+
+	// pre-execute     1~999 (+BaseCode)
+	ErrorPreForceLocal       = BKDistCommonError{Code: BaseCode + 1, Error: DescPreForceLocal}
+	ErrorPreNotSupportRemote = BKDistCommonError{Code: BaseCode + 2, Error: DescPreNotSupportRemote}
+
+	// remote-execute  1000~1999 (+BaseCode)
+	ErrorRemoteSendFile      = BKDistCommonError{Code: BaseCode + 1000, Error: DescRemoteSendFile}
+	ErrorRemoteSendToolchain = BKDistCommonError{Code: BaseCode + 1001, Error: DescRemoteSendToolchain}
+
+	// post-execute    2000~2999 (+BaseCode)
+	ErrorPostMissPumpDependFile   = BKDistCommonError{Code: BaseCode + 2000, Error: DescPostMissPumpDependFile}
+	ErrorPostMissNormalDependFile = BKDistCommonError{Code: BaseCode + 2001, Error: DescPostMissNormalDependFile}
+	ErrorPostSaveFileFailed       = BKDistCommonError{Code: BaseCode + 2002, Error: DescPostSaveFileFailed}
+	ErrorPostOutOfMemoy           = BKDistCommonError{Code: BaseCode + 2100, Error: DescPostOutOfMemoy}
+	ErrorPostToolchainNotFound    = BKDistCommonError{Code: BaseCode + 2200, Error: DescPostToolchainNotFound}
+
+	// local-execute   3000~3999 (+BaseCode)
+
+	// other error     999999
+	ErrorUnknown = BKDistCommonError{Code: 999999, Error: DescUnknown}
+)

src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go

@@ -155,10 +155,10 @@ func (e *executor) executePreTask() (*dcSDK.BKDistCommand, error) {
 
 	dcSDK.StatsTimeNow(&e.stats.PreWorkStartTime)
 	e.mgr.work.Basic().UpdateJobStats(e.stats)
-	r, err := e.handler.PreExecute(e.req.Commands)
+	r, bkerr := e.handler.PreExecute(e.req.Commands)
 	dcSDK.StatsTimeNow(&e.stats.PreWorkEndTime)
-	if err != nil {
-		return nil, err
+	if bkerr.Error != nil {
+		return nil, bkerr.Error
 	}
 
 	e.stats.PreWorkSuccess = true
@@ -211,10 +211,10 @@ func (e *executor) onRemoteFail() (*dcSDK.BKDistCommand, error) {
 
 	// dcSDK.StatsTimeNow(&e.stats.PreWorkStartTime)
 	// e.mgr.work.Basic().UpdateJobStats(e.stats)
-	r, err := e.handler.OnRemoteFail(e.req.Commands)
+	r, bkerr := e.handler.OnRemoteFail(e.req.Commands)
 	// dcSDK.StatsTimeNow(&e.stats.PreWorkEndTime)
-	if err != nil {
-		return nil, err
+	if bkerr.Error != nil {
+		return nil, bkerr.Error
 	}
 
 	// e.stats.PreWorkSuccess = true
@@ -247,9 +247,9 @@ func (e *executor) executePostTask(result *dcSDK.BKDistResult) error {
 	dcSDK.StatsTimeNow(&e.stats.PostWorkStartTime)
 	defer dcSDK.StatsTimeNow(&e.stats.PostWorkEndTime)
 	e.mgr.work.Basic().UpdateJobStats(e.stats)
-	err := e.handler.PostExecute(result)
-	if err != nil {
-		return err
+	bkerr := e.handler.PostExecute(result)
+	if bkerr.Error != nil {
+		return bkerr.Error
 	}
 
 	for i := range result.Results {
@@ -312,7 +312,9 @@ func (e *executor) realExecuteLocalTask(locallockweight int32) *types.LocalTaskE
 	var stdout, stderr []byte
 
 	if e.handler.LocalExecuteNeed(e.req.Commands) {
-		code, err = e.handler.LocalExecute(e.req.Commands)
+		bkerr := e.handler.LocalExecute(e.req.Commands)
+		code = bkerr.Code
+		err = bkerr.Error
 		stdout, stderr = e.Stdout(), e.Stderr()
 	} else {
 		sandbox := e.sandbox.Fork()

src/backend/booster/bk_dist/handler/cc/handler.go

@@ -27,6 +27,7 @@ import (
 	dcPump "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/pump"
 	dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk"
 	dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall"
+	dcType "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types"
 	dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/controller/pkg/manager/analyser"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler"
@@ -150,7 +151,7 @@ func (cc *TaskCC) PreLockWeight(command []string) int32 {
 }
 
 // PreExecute 预处理
-func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, error) {
+func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) {
 	return cc.preExecute(command)
 }
 
@@ -165,8 +166,8 @@ func (cc *TaskCC) LocalLockWeight(command []string) int32 {
 }
 
 // LocalExecute 无需自定义本地处理
-func (cc *TaskCC) LocalExecute(command []string) (int, error) {
-	return 0, nil
+func (cc *TaskCC) LocalExecute(command []string) dcType.BKDistCommonError {
+	return dcType.ErrorNone
 }
 
 // NeedRemoteResource check whether this command need remote resource
@@ -185,7 +186,7 @@ func (cc *TaskCC) NeedRetryOnRemoteFail(command []string) bool {
 }
 
 // OnRemoteFail give chance to try other way if failed to remote execute
-func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) {
+func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) {
 	blog.Infof("cc: start OnRemoteFail for: %v", command)
 
 	if cc.pumpremote {
@@ -195,7 +196,8 @@ func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) {
 		cc.pumpremote = false
 		return cc.preExecute(command)
 	}
-	return nil, nil
+
+	return nil, dcType.ErrorNone
 }
 
 // PostLockWeight decide post-execute lock weight, default 1
@@ -204,7 +206,7 @@ func (cc *TaskCC) PostLockWeight(result *dcSDK.BKDistResult) int32 {
 }
 
 // PostExecute 后置处理, 判断远程执行的结果是否正确
-func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) error {
+func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError {
 	return cc.postExecute(r)
 }
 
@@ -758,7 +760,7 @@ func (cc *TaskCC) workerSupportAbsPath() bool {
 	return true
 }
 
-func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
+func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) {
 	blog.Infof("cc: [%s] start pre execute for: %v", cc.tag, command)
 
 	cc.originArgs = command
@@ -770,25 +772,25 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 		if err != nil {
 			if notifyerr == ErrorNotSupportRemote {
 				blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err)
-				return nil, err
+				return nil, dcType.ErrorPreNotSupportRemote
 			}
 		} else {
 			// for debug
 			blog.Debugf("cc: after try pump, req: %+v", *req)
 			cc.pumpremote = true
-			return req, err
+			return req, dcType.ErrorNone
 		}
 	}
 
 	compilerEnsuredArgs, err := ensureCompiler(command)
 	if err != nil {
 		blog.Warnf("cc: [%s] pre execute ensure compiler %v: %v", cc.tag, command, err)
-		return nil, err
+		return nil, dcType.ErrorUnknown
 	}
 	args, err := expandOptions(cc.sandbox, compilerEnsuredArgs)
 	if err != nil {
 		blog.Warnf("cc: [%s] pre execute expand options %v: %v", cc.tag, compilerEnsuredArgs, err)
-		return nil, err
+		return nil, dcType.ErrorUnknown
 	}
 	cc.ensuredArgs = args
 
@@ -807,7 +809,8 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 		for _, v1 := range cc.ForceLocalCppFileKeys {
 			if v1 != "" && strings.Contains(v, v1) {
 				blog.Warnf("cc: pre execute found %s is in force local list, do not deal now", v)
-				return nil, fmt.Errorf("arg %s is in force local cpp list", v)
+				// return nil, fmt.Errorf("arg %s is in force local cpp list", v)
+				return nil, dcType.ErrorPreForceLocal
 			}
 		}
 	}
@@ -820,7 +823,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 
 	if err = cc.preBuild(args); err != nil {
 		blog.Warnf("cc: [%s] pre execute pre-build %v: %v", cc.tag, args, err)
-		return nil, err
+		return nil, dcType.ErrorUnknown
 	}
 
 	// generate the input files for pre-process file
@@ -829,7 +832,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 		if !existed {
 			err := fmt.Errorf("result file %s not existed", cc.preprocessedFile)
 			blog.Warnf("cc: [%s] %v", cc.tag, err)
-			return nil, err
+			return nil, dcType.ErrorUnknown
 		}
 
 		cc.sendFiles = append(cc.sendFiles, dcSDK.FileDesc{
@@ -870,13 +873,14 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) {
 			},
 		},
 		CustomSave: true,
-	}, nil
+	}, dcType.ErrorNone
 }
 
-func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error {
+func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError {
 	blog.Infof("cc: [%s] start post execute", cc.tag)
 	if r == nil || len(r.Results) == 0 {
-		return ErrorInvalidParam
+		blog.Warnf("cc: [%s] got empty result", cc.tag)
+		return dcType.ErrorUnknown
 	}
 
 	resultfilenum := 0
@@ -893,7 +897,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error {
 			if f.Buffer != nil {
 				if err := saveResultFile(&f, cc.sandbox.Dir); err != nil {
 					blog.Errorf("cc: failed to save file [%s]", f.FilePath)
-					return err
+					return dcType.ErrorPostSaveFileFailed
 				}
 				resultfilenum++
 			}
@@ -914,7 +918,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error {
 		if cc.pumpremote {
 			cc.needcopypumpheadfile = false
 		}
-		return nil
+		return dcType.ErrorNone
 	}
 
 ERROREND:
@@ -939,11 +943,13 @@ ERROREND:
 		os.Remove(cc.pumpHeadFile)
 	}
 
-	return fmt.Errorf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s",
+	blog.Warnf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s",
 		cc.tag,
 		r.Results[0].RetCode,
 		r.Results[0].ErrorMessage,
 		r.Results[0].OutputMessage)
+
+	return dcType.ErrorUnknown
 }
 
 func (cc *TaskCC) ensureOwner(fdl []string) {
@@ -1170,6 +1176,22 @@ func (cc *TaskCC) preBuild(args []string) error {
 				break
 			}
 		}
+
+		// avoid error : does not allow 'register' storage class specifier
+		prefix := "-std=c++"
+		for _, v := range serverSideArgs {
+			if strings.HasPrefix(v, prefix) {
+				if len(v) > len(prefix) {
+					version, err := strconv.Atoi(v[len(prefix):])
+					if err == nil && version > 14 {
+						serverSideArgs = append(serverSideArgs, "-Wno-register")
+						blog.Infof("cc: found %s,ready add [-Wno-register]", v)
+					}
+				}
+				break
+			}
+		}
+
 		cc.serverSideArgs = serverSideArgs
 
 		if cc.isPumpCheck() && pumpErr == nil {

src/backend/booster/bk_dist/handler/custom/handler.go

@@ -135,7 +135,7 @@ func (c *Custom) PreLockWeight(command []string) int32 {
 }
 
 // PreExecute 单个任务的预处理, 如c/c++编译的pre-process, 决定了分发到远程处理的任务信息
-func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, error) {
+func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) {
 	return c.innerHandler.PreExecute(command)
 }
 
@@ -155,7 +155,7 @@ func (c *Custom) LocalLockWeight(command []string) int32 {
 }
 
 // LocalExecute 自定义本地执行
-func (c *Custom) LocalExecute(command []string) (int, error) {
+func (c *Custom) LocalExecute(command []string) dcType.BKDistCommonError {
 	return c.innerHandler.LocalExecute(command)
 }
 
@@ -175,8 +175,8 @@ func (c *Custom) NeedRetryOnRemoteFail(command []string) bool {
 }
 
 // OnRemoteFail give chance to try other way if failed to remote execute
-func (c *Custom) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) {
-	return nil, nil
+func (c *Custom) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) {
+	return nil, dcType.ErrorNone
 }
 
 // PostLockWeight decide post-execute lock weight, default 1
@@ -185,7 +185,7 @@ func (c *Custom) PostLockWeight(result *dcSDK.BKDistResult) int32 {
 }
 
 // PostExecute 单个任务的后置处理, 需要处理远程任务执行的结果
-func (c *Custom) PostExecute(result *dcSDK.BKDistResult) error {
+func (c *Custom) PostExecute(result *dcSDK.BKDistResult) dcType.BKDistCommonError {
 	return c.innerHandler.PostExecute(result)
 }