diff --git a/src/backend/booster/bk_dist/common/types/error.go b/src/backend/booster/bk_dist/common/types/error.go new file mode 100644 index 00000000..f5de3095 --- /dev/null +++ b/src/backend/booster/bk_dist/common/types/error.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2021 THL A29 Limited, a Tencent company. All rights reserved + * + * This source code file is licensed under the MIT License, you may obtain a copy of the License at + * + * http://opensource.org/licenses/MIT + * + */ + +package types + +import "fmt" + +type BKDistCommonError struct { + Code int + Error error +} + +var ( + DescPreForceLocal = fmt.Errorf("cmd in force local when pre execute") + DescPreNotSupportRemote = fmt.Errorf("cmd not support remote when pre execute") + + DescRemoteSendFile = fmt.Errorf("send file failed when remote execute") + DescRemoteSendToolchain = fmt.Errorf("send toolchain failed when remote execute") + + DescPostMissPumpDependFile = fmt.Errorf("miss pump depend file when post execute") + DescPostMissNormalDependFile = fmt.Errorf("miss normal depend file when post execute") + DescPostSaveFileFailed = fmt.Errorf("save file failed when post execute") + DescPostOutOfMemoy = fmt.Errorf("out of memory when post execute") + DescPostToolchainNotFound = fmt.Errorf("toolchain not found when post execute") + + DescUnknown = fmt.Errorf("unknown") +) + +const ( + BaseCode = 100000 +) + +// define errors +var ( + // no error 0 + ErrorNone = BKDistCommonError{Code: 0, Error: nil} + + // pre-execute 1~999 (+BaseCode) + ErrorPreForceLocal = BKDistCommonError{Code: BaseCode + 1, Error: DescPreForceLocal} + ErrorPreNotSupportRemote = BKDistCommonError{Code: BaseCode + 2, Error: DescPreNotSupportRemote} + + // remote-execute 1000~1999 (+BaseCode) + ErrorRemoteSendFile = BKDistCommonError{Code: BaseCode + 1000, Error: DescRemoteSendFile} + ErrorRemoteSendToolchain = BKDistCommonError{Code: BaseCode + 1001, Error: DescRemoteSendToolchain} + + // post-execute 2000~2999 (+BaseCode) + ErrorPostMissPumpDependFile = BKDistCommonError{Code: BaseCode + 2000, Error: DescPostMissPumpDependFile} + ErrorPostMissNormalDependFile = BKDistCommonError{Code: BaseCode + 2001, Error: DescPostMissNormalDependFile} + ErrorPostSaveFileFailed = BKDistCommonError{Code: BaseCode + 2002, Error: DescPostSaveFileFailed} + ErrorPostOutOfMemoy = BKDistCommonError{Code: BaseCode + 2100, Error: DescPostOutOfMemoy} + ErrorPostToolchainNotFound = BKDistCommonError{Code: BaseCode + 2200, Error: DescPostToolchainNotFound} + + // local-execute 3000~3999 (+BaseCode) + + // other error 999999 + ErrorUnknown = BKDistCommonError{Code: 999999, Error: DescUnknown} +) diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go index d6ac105a..b62f634f 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go @@ -155,10 +155,10 @@ func (e *executor) executePreTask() (*dcSDK.BKDistCommand, error) { dcSDK.StatsTimeNow(&e.stats.PreWorkStartTime) e.mgr.work.Basic().UpdateJobStats(e.stats) - r, err := e.handler.PreExecute(e.req.Commands) + r, bkerr := e.handler.PreExecute(e.req.Commands) dcSDK.StatsTimeNow(&e.stats.PreWorkEndTime) - if err != nil { - return nil, err + if bkerr.Error != nil { + return nil, bkerr.Error } e.stats.PreWorkSuccess = true @@ -211,10 +211,10 @@ func (e *executor) onRemoteFail() (*dcSDK.BKDistCommand, error) { // dcSDK.StatsTimeNow(&e.stats.PreWorkStartTime) // e.mgr.work.Basic().UpdateJobStats(e.stats) - r, err := e.handler.OnRemoteFail(e.req.Commands) + r, bkerr := e.handler.OnRemoteFail(e.req.Commands) // dcSDK.StatsTimeNow(&e.stats.PreWorkEndTime) - if err != nil { - return nil, err + if bkerr.Error != nil { + return nil, bkerr.Error } // e.stats.PreWorkSuccess = true @@ -247,9 +247,9 @@ func (e *executor) executePostTask(result *dcSDK.BKDistResult) error { dcSDK.StatsTimeNow(&e.stats.PostWorkStartTime) defer dcSDK.StatsTimeNow(&e.stats.PostWorkEndTime) e.mgr.work.Basic().UpdateJobStats(e.stats) - err := e.handler.PostExecute(result) - if err != nil { - return err + bkerr := e.handler.PostExecute(result) + if bkerr.Error != nil { + return bkerr.Error } for i := range result.Results { @@ -312,7 +312,9 @@ func (e *executor) realExecuteLocalTask(locallockweight int32) *types.LocalTaskE var stdout, stderr []byte if e.handler.LocalExecuteNeed(e.req.Commands) { - code, err = e.handler.LocalExecute(e.req.Commands) + bkerr := e.handler.LocalExecute(e.req.Commands) + code = bkerr.Code + err = bkerr.Error stdout, stderr = e.Stdout(), e.Stderr() } else { sandbox := e.sandbox.Fork() diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go index 1b36301d..2f6a2327 100644 --- a/src/backend/booster/bk_dist/handler/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/cc/handler.go @@ -27,6 +27,7 @@ import ( dcPump "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/pump" dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" + dcType "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types" dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/controller/pkg/manager/analyser" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler" @@ -150,7 +151,7 @@ func (cc *TaskCC) PreLockWeight(command []string) int32 { } // PreExecute 预处理 -func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return cc.preExecute(command) } @@ -165,8 +166,8 @@ func (cc *TaskCC) LocalLockWeight(command []string) int32 { } // LocalExecute 无需自定义本地处理 -func (cc *TaskCC) LocalExecute(command []string) (int, error) { - return 0, nil +func (cc *TaskCC) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // NeedRemoteResource check whether this command need remote resource @@ -185,7 +186,7 @@ func (cc *TaskCC) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { +func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("cc: start OnRemoteFail for: %v", command) if cc.pumpremote { @@ -195,7 +196,8 @@ func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { cc.pumpremote = false return cc.preExecute(command) } - return nil, nil + + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -204,7 +206,7 @@ func (cc *TaskCC) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理, 判断远程执行的结果是否正确 -func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) error { +func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { return cc.postExecute(r) } @@ -758,7 +760,7 @@ func (cc *TaskCC) workerSupportAbsPath() bool { return true } -func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("cc: [%s] start pre execute for: %v", cc.tag, command) cc.originArgs = command @@ -770,25 +772,25 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err != nil { if notifyerr == ErrorNotSupportRemote { blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err) - return nil, err + return nil, dcType.ErrorPreNotSupportRemote } } else { // for debug blog.Debugf("cc: after try pump, req: %+v", *req) cc.pumpremote = true - return req, err + return req, dcType.ErrorNone } } compilerEnsuredArgs, err := ensureCompiler(command) if err != nil { blog.Warnf("cc: [%s] pre execute ensure compiler %v: %v", cc.tag, command, err) - return nil, err + return nil, dcType.ErrorUnknown } args, err := expandOptions(cc.sandbox, compilerEnsuredArgs) if err != nil { blog.Warnf("cc: [%s] pre execute expand options %v: %v", cc.tag, compilerEnsuredArgs, err) - return nil, err + return nil, dcType.ErrorUnknown } cc.ensuredArgs = args @@ -807,7 +809,8 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { for _, v1 := range cc.ForceLocalCppFileKeys { if v1 != "" && strings.Contains(v, v1) { blog.Warnf("cc: pre execute found %s is in force local list, do not deal now", v) - return nil, fmt.Errorf("arg %s is in force local cpp list", v) + // return nil, fmt.Errorf("arg %s is in force local cpp list", v) + return nil, dcType.ErrorPreForceLocal } } } @@ -820,7 +823,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err = cc.preBuild(args); err != nil { blog.Warnf("cc: [%s] pre execute pre-build %v: %v", cc.tag, args, err) - return nil, err + return nil, dcType.ErrorUnknown } // generate the input files for pre-process file @@ -829,7 +832,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if !existed { err := fmt.Errorf("result file %s not existed", cc.preprocessedFile) blog.Warnf("cc: [%s] %v", cc.tag, err) - return nil, err + return nil, dcType.ErrorUnknown } cc.sendFiles = append(cc.sendFiles, dcSDK.FileDesc{ @@ -870,13 +873,14 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { }, }, CustomSave: true, - }, nil + }, dcType.ErrorNone } -func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { +func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("cc: [%s] start post execute", cc.tag) if r == nil || len(r.Results) == 0 { - return ErrorInvalidParam + blog.Warnf("cc: [%s] got empty result", cc.tag) + return dcType.ErrorUnknown } resultfilenum := 0 @@ -893,7 +897,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { if f.Buffer != nil { if err := saveResultFile(&f, cc.sandbox.Dir); err != nil { blog.Errorf("cc: failed to save file [%s]", f.FilePath) - return err + return dcType.ErrorPostSaveFileFailed } resultfilenum++ } @@ -914,7 +918,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { if cc.pumpremote { cc.needcopypumpheadfile = false } - return nil + return dcType.ErrorNone } ERROREND: @@ -939,11 +943,13 @@ ERROREND: os.Remove(cc.pumpHeadFile) } - return fmt.Errorf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s", + blog.Warnf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s", cc.tag, r.Results[0].RetCode, r.Results[0].ErrorMessage, r.Results[0].OutputMessage) + + return dcType.ErrorUnknown } func (cc *TaskCC) ensureOwner(fdl []string) { @@ -1170,6 +1176,22 @@ func (cc *TaskCC) preBuild(args []string) error { break } } + + // avoid error : does not allow 'register' storage class specifier + prefix := "-std=c++" + for _, v := range serverSideArgs { + if strings.HasPrefix(v, prefix) { + if len(v) > len(prefix) { + version, err := strconv.Atoi(v[len(prefix):]) + if err == nil && version > 14 { + serverSideArgs = append(serverSideArgs, "-Wno-register") + blog.Infof("cc: found %s,ready add [-Wno-register]", v) + } + } + break + } + } + cc.serverSideArgs = serverSideArgs if cc.isPumpCheck() && pumpErr == nil { diff --git a/src/backend/booster/bk_dist/handler/custom/handler.go b/src/backend/booster/bk_dist/handler/custom/handler.go index c72f025b..50183392 100644 --- a/src/backend/booster/bk_dist/handler/custom/handler.go +++ b/src/backend/booster/bk_dist/handler/custom/handler.go @@ -135,7 +135,7 @@ func (c *Custom) PreLockWeight(command []string) int32 { } // PreExecute 单个任务的预处理, 如c/c++编译的pre-process, 决定了分发到远程处理的任务信息 -func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (c *Custom) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return c.innerHandler.PreExecute(command) } @@ -155,7 +155,7 @@ func (c *Custom) LocalLockWeight(command []string) int32 { } // LocalExecute 自定义本地执行 -func (c *Custom) LocalExecute(command []string) (int, error) { +func (c *Custom) LocalExecute(command []string) dcType.BKDistCommonError { return c.innerHandler.LocalExecute(command) } @@ -175,8 +175,8 @@ func (c *Custom) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (c *Custom) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (c *Custom) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -185,7 +185,7 @@ func (c *Custom) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 单个任务的后置处理, 需要处理远程任务执行的结果 -func (c *Custom) PostExecute(result *dcSDK.BKDistResult) error { +func (c *Custom) PostExecute(result *dcSDK.BKDistResult) dcType.BKDistCommonError { return c.innerHandler.PostExecute(result) } diff --git a/src/backend/booster/bk_dist/handler/echo/handler.go b/src/backend/booster/bk_dist/handler/echo/handler.go index 8f7462f7..d29597eb 100644 --- a/src/backend/booster/bk_dist/handler/echo/handler.go +++ b/src/backend/booster/bk_dist/handler/echo/handler.go @@ -10,7 +10,6 @@ package echo import ( - "fmt" "os" "os/exec" "path/filepath" @@ -129,8 +128,8 @@ func (c *Echo) LocalLockWeight(command []string) int32 { } // LocalExecute execute local cmd by this handle -func (c *Echo) LocalExecute(command []string) (int, error) { - return 0, nil +func (c *Echo) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // PreLockWeight decide pre-execute lock weight, default 1 @@ -140,11 +139,12 @@ func (c *Echo) PreLockWeight(command []string) int32 { // PreExecute do the pre-process in one executor command. // PreExecute should analyse the input command and generate the DistCommand to send to remote. -func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { upperCommand := strings.ToUpper(command[0]) if len(command) < 2 || (!strings.HasSuffix(upperCommand, "ECHO") && !strings.HasSuffix(upperCommand, "ECHO.EXE")) { - return nil, fmt.Errorf("invalid command,len(command):[%d], command[0]:[%s]", len(command), command[0]) + blog.Warnf("echo: invalid command,len(command):[%d], command[0]:[%s]", len(command), command[0]) + return nil, dcType.ErrorUnknown } inputfiles := make([]dcSDK.FileDesc, 0, 1) @@ -165,7 +165,7 @@ func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { existed, fileSize, modifyTime, fileMode = dcFile.Stat(newinput).Batch() if !existed { blog.Infof("echo: input file %s not exist", newinput) - return nil, fmt.Errorf("echo: input file %s not exist", command[i]) + return nil, dcType.ErrorUnknown } } } @@ -194,7 +194,7 @@ func (c *Echo) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { }, }, CustomSave: true, - }, nil + }, dcType.ErrorNone } // NeedRemoteResource check whether this command need remote resource @@ -213,8 +213,8 @@ func (c *Echo) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (c *Echo) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (c *Echo) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -225,24 +225,27 @@ func (c *Echo) PostLockWeight(result *dcSDK.BKDistResult) int32 { // PostExecute do the post-process in one executor command. // PostExecute should check the DistResult and judge whether the remote processing succeeded. // Also PostExecute should manages the output message. -func (c *Echo) PostExecute(r *dcSDK.BKDistResult) error { +func (c *Echo) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { // do not save result to disk if r == nil || len(r.Results) == 0 { - return fmt.Errorf("echo: result data is invalid") + blog.Warnf("echo: result data is invalid") + return dcType.ErrorUnknown } result := r.Results[0] if len(result.ResultFiles) == 0 { - return fmt.Errorf("echo: not found result file, retcode %d, error message:[%s], output message:[%s]", + blog.Warnf("echo: not found result file, retcode %d, error message:[%s], output message:[%s]", result.RetCode, result.ErrorMessage, result.OutputMessage) + + return dcType.ErrorUnknown } for _, v := range result.ResultFiles { blog.Infof("echo: received result file[%s],len[%d]", v.FilePath, len(v.Buffer)) } - return nil + return dcType.ErrorNone } // FinalExecute provide a chance to do process before the process exit. diff --git a/src/backend/booster/bk_dist/handler/find/handler.go b/src/backend/booster/bk_dist/handler/find/handler.go index d7c2fdc6..b656f8d1 100644 --- a/src/backend/booster/bk_dist/handler/find/handler.go +++ b/src/backend/booster/bk_dist/handler/find/handler.go @@ -21,6 +21,7 @@ import ( dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" dcType "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler" + "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/codec" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/server/pkg/engine/disttask" ) @@ -50,9 +51,10 @@ func NewFinder() (handler.Handler, error) { // 3. do PreWork before all process begin // 4. do the Command from user, when the user command running, multiple "find" commands brings up. // 5. in each "find" command: -// - do PreExecute -// - run command (in local or remote) -// - do PostExecute +// - do PreExecute +// - run command (in local or remote) +// - do PostExecute +// // 6. do PostWork after all process finish // // Particularly, MParallel.exe is recommended to used to manage the parallel process for "find" commands. @@ -70,8 +72,8 @@ func NewFinder() (handler.Handler, error) { // test1.txt:23 ----------TEST1.TXT: 23 // // test2.txt: foobar ----------TEST2.TXT: -// foobar // +// foobar type Finder struct { sandbox *dcSyscall.Sandbox inputFiles []string @@ -171,8 +173,8 @@ func (c *Finder) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (c *Finder) LocalExecute(command []string) (int, error) { - return 0, nil +func (c *Finder) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // PreLockWeight decide pre-execute lock weight, default 1 @@ -182,13 +184,14 @@ func (c *Finder) PreLockWeight(command []string) int32 { // PreExecute do the pre-process in one executor command. // PreExecute should analyse the input command and generate the DistCommand to send to remote. -func (c *Finder) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (c *Finder) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { // command must be "find" //if len(command) < 3 || (strings.ToUpper(command[0]) != "FIND" && strings.ToUpper(command[0]) != "FIND.EXE") { upperCommand := strings.ToUpper(command[0]) if len(command) < 3 || (!strings.HasSuffix(upperCommand, "FIND") && !strings.HasSuffix(upperCommand, "FIND.EXE")) { - return nil, fmt.Errorf("invalid command,len(command):[%d], command[0]:[%s]", len(command), command[0]) + blog.Warnf("find: invalid command,len(command):[%d], command[0]:[%s]", len(command), command[0]) + return nil, dcType.ErrorUnknown } strSeen := false @@ -226,7 +229,8 @@ func (c *Finder) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { // all the others are input files. existed, fileSize, modifyTime, fileMode := dcFile.Stat(arg).Batch() if !existed { - return nil, fmt.Errorf("input file %s not exist", arg) + blog.Warnf("find: input file %s not exist", arg) + return nil, dcType.ErrorUnknown } inputFiles = append(inputFiles, dcSDK.FileDesc{ FilePath: arg, @@ -252,7 +256,7 @@ func (c *Finder) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { Inputfiles: inputFiles, }, }, - }, nil + }, dcType.ErrorNone } // NeedRemoteResource check whether this command need remote resource @@ -271,8 +275,8 @@ func (c *Finder) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (c *Finder) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (c *Finder) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -283,14 +287,16 @@ func (c *Finder) PostLockWeight(result *dcSDK.BKDistResult) int32 { // PostExecute do the post-process in one executor command. // PostExecute should check the DistResult and judge whether the remote processing succeeded. // Also PostExecute should manages the output message. -func (c *Finder) PostExecute(r *dcSDK.BKDistResult) error { +func (c *Finder) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { if r == nil || len(r.Results) == 0 { - return fmt.Errorf("invalid param") + blog.Warnf("find: parameter is invalid") + return dcType.ErrorUnknown } result := r.Results[0] if result.RetCode != 0 { - return fmt.Errorf("failed to execute on remote: %s", string(result.ErrorMessage)) + blog.Warnf("find: failed to execute on remote: %s", string(result.ErrorMessage)) + return dcType.ErrorUnknown } belongs := "" @@ -314,7 +320,7 @@ func (c *Finder) PostExecute(r *dcSDK.BKDistResult) error { fmt.Printf("%s\n", originResult) } - return nil + return dcType.ErrorNone } // FinalExecute provide a chance to do process before the process exit. diff --git a/src/backend/booster/bk_dist/handler/handler.go b/src/backend/booster/bk_dist/handler/handler.go index cd90dff0..ac109f9e 100644 --- a/src/backend/booster/bk_dist/handler/handler.go +++ b/src/backend/booster/bk_dist/handler/handler.go @@ -47,7 +47,7 @@ type Handler interface { PreLockWeight(command []string) int32 // PreExecute will be called before task is distributed - PreExecute(command []string) (*dcSDK.BKDistCommand, error) + PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) // LocalExecuteNeed decide whether executor should execute local command LocalExecuteNeed(command []string) bool @@ -56,7 +56,7 @@ type Handler interface { LocalLockWeight(command []string) int32 // LocalExecute will execute this command by handler - LocalExecute(command []string) (int, error) + LocalExecute(command []string) dcType.BKDistCommonError // NeedRemoteResource check whether this command need remote resource NeedRemoteResource(command []string) bool @@ -68,7 +68,7 @@ type Handler interface { NeedRetryOnRemoteFail(command []string) bool // OnRemoteFail give chance to try other way if failed to remote execute - OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) + OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) // PostExecuteNeedLock decide whether executor should lock before post execution PostExecuteNeedLock(result *dcSDK.BKDistResult) bool @@ -77,7 +77,7 @@ type Handler interface { PostLockWeight(result *dcSDK.BKDistResult) int32 // PostExecute will be called after task is distributed and executed - PostExecute(result *dcSDK.BKDistResult) error + PostExecute(result *dcSDK.BKDistResult) dcType.BKDistCommonError // FinalExecute chance to finalize for handler, must be safe to call in goroutines FinalExecute(command []string) diff --git a/src/backend/booster/bk_dist/handler/tc/handler.go b/src/backend/booster/bk_dist/handler/tc/handler.go index 8cc25e95..de685ecf 100644 --- a/src/backend/booster/bk_dist/handler/tc/handler.go +++ b/src/backend/booster/bk_dist/handler/tc/handler.go @@ -19,6 +19,7 @@ import ( dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" dcType "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler" + "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" ) // NewTextureCompressor get a new tc handler @@ -157,28 +158,33 @@ func (tc *TextureCompressor) PreLockWeight(command []string) int32 { } // PreExecute parse the input and output file, and then just run the origin command in remote -func (tc *TextureCompressor) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (tc *TextureCompressor) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { if len(command) == 0 { - return nil, fmt.Errorf("invalid command") + blog.Warnf("tc: invalid command") + return nil, dcType.ErrorUnknown } t, err := getTCType(command[0]) if err != nil { - return nil, err + blog.Warnf("tc: get tc type with error:%v", err) + return nil, dcType.ErrorUnknown } inputFile, err := t.getInputFile(command[1:]) if err != nil { - return nil, err + blog.Warnf("tc: get tc input file with error:%v", err) + return nil, dcType.ErrorUnknown } outputFile, err := t.getOutputFile(command[1:]) if err != nil { - return nil, err + blog.Warnf("tc: get tc output file with error:%v", err) + return nil, dcType.ErrorUnknown } existed, fileSize, modifyTime, fileMode := dcFile.Stat(inputFile).Batch() if !existed { - return nil, fmt.Errorf("input file %s not exist", inputFile) + blog.Warnf("tc: input file %s not exist", inputFile) + return nil, dcType.ErrorUnknown } return &dcSDK.BKDistCommand{ @@ -199,7 +205,7 @@ func (tc *TextureCompressor) PreExecute(command []string) (*dcSDK.BKDistCommand, ResultFiles: []string{outputFile}, }, }, - }, nil + }, dcType.ErrorNone } // NeedRemoteResource check whether this command need remote resource @@ -218,8 +224,8 @@ func (tc *TextureCompressor) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (tc *TextureCompressor) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (tc *TextureCompressor) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -228,17 +234,19 @@ func (tc *TextureCompressor) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute judge the result -func (tc *TextureCompressor) PostExecute(r *dcSDK.BKDistResult) error { +func (tc *TextureCompressor) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { if r == nil || len(r.Results) == 0 { - return fmt.Errorf("invalid param") + blog.Warnf("tc: parameter is invalid") + return dcType.ErrorUnknown } result := r.Results[0] if result.RetCode != 0 { - return fmt.Errorf("failed to execute on remote: %s", string(result.ErrorMessage)) + blog.Warnf("tc: failed to execute on remote: %s", string(result.ErrorMessage)) + return dcType.ErrorUnknown } - return nil + return dcType.ErrorNone } // LocalExecuteNeed no need @@ -252,8 +260,8 @@ func (tc *TextureCompressor) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (tc *TextureCompressor) LocalExecute(command []string) (int, error) { - return 0, nil +func (tc *TextureCompressor) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // FinalExecute no need diff --git a/src/backend/booster/bk_dist/handler/ue4/astc/handler.go b/src/backend/booster/bk_dist/handler/ue4/astc/handler.go index a640ee2d..4899cdf3 100644 --- a/src/backend/booster/bk_dist/handler/ue4/astc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/astc/handler.go @@ -136,29 +136,33 @@ func (tc *TextureCompressor) PreLockWeight(command []string) int32 { } // PreExecute parse the input and output file, and then just run the origin command in remote -func (tc *TextureCompressor) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (tc *TextureCompressor) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { if len(command) == 0 { - return nil, fmt.Errorf("invalid command") + blog.Warnf("astc: invalid command") + return nil, dcType.ErrorUnknown } - blog.Infof("tc: pre execute for: %v", command) + blog.Infof("astc: pre execute for: %v", command) t, err := getTCType(command[0]) if err != nil { - return nil, err + blog.Warnf("astc: get tc type with error:%v", err) + return nil, dcType.ErrorUnknown } var inputFile string var newparam []string inputFile, tc.outputTempFile, tc.outputRealFile, newparam, err = t.scanParam(command[1:]) if err != nil { - return nil, err + blog.Warnf("astc: scan param with error:%v", err) + return nil, dcType.ErrorUnknown } - blog.Infof("tc: got new param [%s]", strings.Join(newparam, " ")) + blog.Infof("astc: got new param [%s]", strings.Join(newparam, " ")) existed, fileSize, modifyTime, fileMode := dcFile.Stat(inputFile).Batch() if !existed { - return nil, fmt.Errorf("input file %s not exist", inputFile) + blog.Warnf("astc: input file %s not exist", inputFile) + return nil, dcType.ErrorUnknown } return &dcSDK.BKDistCommand{ @@ -182,7 +186,7 @@ func (tc *TextureCompressor) PreExecute(command []string) (*dcSDK.BKDistCommand, ResultFiles: []string{tc.outputTempFile}, }, }, - }, nil + }, dcType.ErrorNone } // RemoteRetryTimes will return the remote retry times @@ -196,8 +200,8 @@ func (tc *TextureCompressor) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (tc *TextureCompressor) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (tc *TextureCompressor) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -206,28 +210,31 @@ func (tc *TextureCompressor) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute judge the result -func (tc *TextureCompressor) PostExecute(r *dcSDK.BKDistResult) error { +func (tc *TextureCompressor) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { if r == nil || len(r.Results) == 0 { - return fmt.Errorf("invalid param") + blog.Warnf("astc: parameter is invalid") + return dcType.ErrorUnknown } result := r.Results[0] if result.RetCode != 0 { - return fmt.Errorf("failed to execute on remote: %s", string(result.ErrorMessage)) + blog.Warnf("astc: failed to execute on remote: %s", string(result.ErrorMessage)) + return dcType.ErrorUnknown } if len(result.ResultFiles) == 0 { - return fmt.Errorf("tc: not found result file, retcode %d, error message:[%s], output message:[%s]", + blog.Warnf("astc: not found result file, retcode %d, error message:[%s], output message:[%s]", result.RetCode, result.ErrorMessage, result.OutputMessage) + return dcType.ErrorUnknown } // move result temp to real - blog.Infof("tc: ready rename file from [%s] to [%s]", tc.outputTempFile, tc.outputRealFile) + blog.Infof("astc: ready rename file from [%s] to [%s]", tc.outputTempFile, tc.outputRealFile) _ = os.Rename(tc.outputTempFile, tc.outputRealFile) - return nil + return dcType.ErrorNone } // LocalExecuteNeed no need @@ -246,8 +253,8 @@ func (tc *TextureCompressor) NeedRemoteResource(command []string) bool { } // LocalExecute no need -func (tc *TextureCompressor) LocalExecute(command []string) (int, error) { - return 0, nil +func (tc *TextureCompressor) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // FinalExecute no need diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go index 53e08ae7..5ffa3030 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go @@ -28,6 +28,7 @@ import ( dcPump "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/pump" dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" + dcType "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types" dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util" commonUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler/common" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" @@ -141,7 +142,7 @@ func (cc *TaskCC) PreLockWeight(command []string) int32 { } // PreExecute 预处理 -func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cc *TaskCC) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return cc.preExecute(command) } @@ -156,8 +157,8 @@ func (cc *TaskCC) LocalLockWeight(command []string) int32 { } // LocalExecute 无需自定义本地处理 -func (cc *TaskCC) LocalExecute(command []string) (int, error) { - return 0, nil +func (cc *TaskCC) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // NeedRemoteResource check whether this command need remote resource @@ -176,7 +177,7 @@ func (cc *TaskCC) NeedRetryOnRemoteFail(command []string) bool { } // TODO : OnRemoteFail give chance to try other way if failed to remote execute -func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { +func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("cc: start OnRemoteFail for: %v", command) if cc.pumpremote { @@ -186,7 +187,8 @@ func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { cc.pumpremote = false return cc.preExecute(command) } - return nil, nil + + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -195,7 +197,7 @@ func (cc *TaskCC) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理, 判断远程执行的结果是否正确 -func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) error { +func (cc *TaskCC) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { return cc.postExecute(r) } @@ -882,7 +884,7 @@ func (cc *TaskCC) workerSupportAbsPath() bool { return true } -func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("cc: start pre execute for: %v", command) // debugRecordFileName(fmt.Sprintf("cc: start pre execute for: %v", command)) @@ -896,13 +898,13 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err != nil { if notifyerr == ErrorNotSupportRemote { blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err) - return nil, err + return nil, dcType.ErrorUnknown } } else { // for debug blog.Debugf("cc: after try pump, req: %+v", *req) cc.pumpremote = true - return req, err + return req, dcType.ErrorNone } } } @@ -911,7 +913,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { responseFile, args, err := ensureCompiler(command, cc.sandbox.Dir) if err != nil { blog.Warnf("cc: pre execute ensure compiler %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } // obtain force key set by booster @@ -932,7 +934,8 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if v != "" && strings.Contains(responseFile, v) { blog.Warnf("cc: pre execute found response %s is in force local list, do not deal now", responseFile) - return nil, fmt.Errorf("response file %s is in force local list", responseFile) + // blog.Warnf("response file %s is in force local list", responseFile) + return nil, dcType.ErrorPreForceLocal } } } @@ -942,7 +945,8 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { for _, v1 := range cc.ForceLocalCppFileKeys { if v1 != "" && strings.Contains(v, v1) { blog.Warnf("cc: pre execute found %s is in force local list, do not deal now", v) - return nil, fmt.Errorf("arg %s is in force local cpp list", v) + // return nil, fmt.Errorf("arg %s is in force local cpp list", v) + return nil, dcType.ErrorPreForceLocal } } break @@ -962,7 +966,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err = cc.preBuild(args); err != nil { blog.Warnf("cc: pre execute pre-build %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } // debugRecordFileName("FileInfo begin") @@ -971,7 +975,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if !existed { err := fmt.Errorf("result file %s not existed", cc.preprocessedFile) blog.Errorf("%v", err) - return nil, err + return nil, dcType.ErrorUnknown } // generate the input files for pre-process file @@ -1013,13 +1017,14 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { }, }, CustomSave: true, - }, nil + }, dcType.ErrorNone } -func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { +func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("cc: start post execute for: %v", cc.originArgs) if r == nil || len(r.Results) == 0 { - return ErrorInvalidParam + blog.Warnf("cc: parameter is invalid") + return dcType.ErrorUnknown } resultfilenum := 0 @@ -1035,8 +1040,8 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { for _, f := range r.Results[0].ResultFiles { if f.Buffer != nil { if err := saveResultFile(&f, cc.sandbox.Dir); err != nil { - blog.Errorf("cc: failed to save file [%s]", f.FilePath) - return err + blog.Errorf("cc: failed to save file [%s] with error:%v", f.FilePath, err) + return dcType.ErrorUnknown } resultfilenum++ } @@ -1057,7 +1062,7 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { if cc.pumpremote { cc.needcopypumpheadfile = false } - return nil + return dcType.ErrorNone } ERROREND: @@ -1081,10 +1086,11 @@ ERROREND: os.Remove(cc.pumpHeadFile) } - return fmt.Errorf("cc: failed to remote execute, retcode %d, error message:%s, output message:%s", + blog.Warnf("cc: failed to remote execute, retcode %d, error message:%s, output message:%s", r.Results[0].RetCode, r.Results[0].ErrorMessage, r.Results[0].OutputMessage) + return dcType.ErrorUnknown } func (cc *TaskCC) resolvePchDepend(workdir string) error { @@ -1268,6 +1274,21 @@ func (cc *TaskCC) preBuild(args []string) error { } } + // avoid error : does not allow 'register' storage class specifier + prefix := "-std=c++" + for _, v := range serverSideArgs { + if strings.HasPrefix(v, prefix) { + if len(v) > len(prefix) { + version, err := strconv.Atoi(v[len(prefix):]) + if err == nil && version > 14 { + serverSideArgs = append(serverSideArgs, "-Wno-register") + blog.Infof("cc: found %s,ready add [-Wno-register]", v) + } + } + break + } + } + // quota result file if it's path contains space if runtime.GOOS == osWindows { if hasSpace(cc.outputFile) && !strings.HasPrefix(cc.outputFile, "\"") { diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go index 9d104106..2828787a 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go @@ -245,7 +245,7 @@ func (cl *TaskCL) PreLockWeight(command []string) int32 { } // PreExecute 预处理 -func (cl *TaskCL) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cl *TaskCL) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return cl.preExecute(command) } @@ -265,7 +265,7 @@ func (cl *TaskCL) NeedRetryOnRemoteFail(command []string) bool { } // TODO : OnRemoteFail give chance to try other way if failed to remote execute -func (cl *TaskCL) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { +func (cl *TaskCL) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("cl: start OnRemoteFail for: %v", command) if cl.pumpremote { @@ -275,7 +275,7 @@ func (cl *TaskCL) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { cl.pumpremote = false return cl.preExecute(command) } - return nil, nil + return nil, dcType.ErrorNone } // LocalLockWeight decide local-execute lock weight, default 1 @@ -284,7 +284,7 @@ func (cl *TaskCL) LocalLockWeight(command []string) int32 { } // PostExecute 后置处理 -func (cl *TaskCL) PostExecute(r *dcSDK.BKDistResult) error { +func (cl *TaskCL) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { return cl.postExecute(r) } @@ -299,17 +299,20 @@ func (cl *TaskCL) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // LocalExecute no need -func (cl *TaskCL) LocalExecute(command []string) (int, error) { +func (cl *TaskCL) LocalExecute(command []string) dcType.BKDistCommonError { if len(command) < 1 { - return 0, fmt.Errorf("cl: failed to execute command, for args is empty") + blog.Warnf("cl: failed to execute command, for args is empty") + return dcType.ErrorUnknown } sandbox := cl.sandbox.Fork() flag, rspfile, err := cl.needSaveResponseFile(command) if flag && err == nil { - return sandbox.ExecCommand(command[0], fmt.Sprintf("@%s", rspfile)) + code, err := sandbox.ExecCommand(command[0], fmt.Sprintf("@%s", rspfile)) + return dcType.BKDistCommonError{Code: code, Error: err} } else { - return sandbox.ExecCommand(command[0], command[1:]...) + code, err := sandbox.ExecCommand(command[0], command[1:]...) + return dcType.BKDistCommonError{Code: code, Error: err} } } @@ -828,7 +831,7 @@ func (cl *TaskCL) workerSupportAbsPath() bool { return true } -func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("cl: start pre execute for: %v", command) // debugRecordFileName(fmt.Sprintf("cl: start pre execute for: %v", command)) @@ -842,11 +845,11 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err != nil { if notifyerr == ErrorNotSupportRemote { blog.Warnf("cl: pre execute failed to try pump %v: %v", command, err) - return nil, err + return nil, dcType.ErrorUnknown } } else { cl.pumpremote = true - return req, err + return req, dcType.ErrorNone } } } @@ -855,7 +858,7 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { responseFile, args, showinclude, err := ensureCompiler(command, cl.sandbox.Dir) if err != nil { blog.Warnf("cl: pre execute ensure compiler failed %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } // obtain force key set by booster @@ -879,7 +882,7 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if v != "" && strings.Contains(responseFile, v) { blog.Warnf("cl: pre execute found response %s is in force local list, do not deal now", responseFile) - return nil, fmt.Errorf("response file %s is in force local list", responseFile) + return nil, dcType.ErrorPreForceLocal } } } @@ -890,7 +893,7 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { for _, v1 := range cl.ForceLocalCppFileKeys { if v1 != "" && strings.Contains(v, v1) { blog.Warnf("cl: pre execute found %s is in force local list, do not deal now", v) - return nil, fmt.Errorf("arg %s is in force local cpp list", v) + return nil, dcType.ErrorPreForceLocal } } break @@ -911,7 +914,7 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err = cl.preBuild(args); err != nil { blog.Debugf("cl: pre execute pre-build %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } tend := time.Now().Local() @@ -930,9 +933,8 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { } else { existed, fileSize, modifyTime, fileMode = dcFile.Stat(cl.preprocessedFile).Batch() if !existed { - err := fmt.Errorf("input pre file %s not existed", cl.preprocessedFile) - blog.Errorf("%v", err) - return nil, err + blog.Errorf("cl: input pre file %s not existed", cl.preprocessedFile) + return nil, dcType.ErrorUnknown } } @@ -966,9 +968,8 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { params = []string{fmt.Sprintf("@%s", rspfile)} existed, fileSize, modifyTime, fileMode := dcFile.Stat(rspfile).Batch() if !existed { - err := fmt.Errorf("input response file %s not existed", rspfile) - blog.Errorf("%v", err) - return nil, err + blog.Errorf("cl: input response file %s not existed", rspfile) + return nil, dcType.ErrorUnknown } inputFiles = append(inputFiles, dcSDK.FileDesc{ FilePath: rspfile, @@ -996,13 +997,14 @@ func (cl *TaskCL) preExecute(command []string) (*dcSDK.BKDistCommand, error) { }, }, CustomSave: true, - }, nil + }, dcType.ErrorNone } -func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) error { +func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("cl: start post execute for: %v", cl.originArgs) if r == nil || len(r.Results) == 0 { - return ErrorInvalidParam + blog.Warnf("cl: parameter is invalid") + return dcType.ErrorUnknown } resultfilenum := 0 @@ -1019,7 +1021,7 @@ func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) error { if f.Buffer != nil { if err := saveResultFile(&f, cl.sandbox.Dir); err != nil { blog.Errorf("cl: failed to save file [%s]", f.FilePath) - return err + return dcType.ErrorUnknown } resultfilenum++ } @@ -1056,7 +1058,7 @@ func (cl *TaskCL) postExecute(r *dcSDK.BKDistResult) error { cl.needcopypumpheadfile = false } - return nil + return dcType.ErrorNone } ERROREND: @@ -1080,10 +1082,12 @@ ERROREND: os.Remove(cl.pumpHeadFile) } - return fmt.Errorf("cl: failed to remote execute, retcode %d, error message:%s, output message:%s", + blog.Warnf("cl: failed to remote execute, retcode %d, error message:%s, output message:%s", r.Results[0].RetCode, r.Results[0].ErrorMessage, r.Results[0].OutputMessage) + + return dcType.ErrorUnknown } func (cl *TaskCL) finalExecute([]string) { diff --git a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go index 74dd31ac..79af3a1b 100644 --- a/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go @@ -109,7 +109,7 @@ func (cf *TaskCLFilter) PreLockWeight(command []string) int32 { } // PreExecute 预处理, 复用cl-handler的逻辑 -func (cf *TaskCLFilter) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cf *TaskCLFilter) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return cf.preExecute(command) } @@ -133,12 +133,12 @@ func (cf *TaskCLFilter) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (cf *TaskCLFilter) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { +func (cf *TaskCLFilter) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { if cf.clhandle != nil { return cf.clhandle.OnRemoteFail(cf.cldArgs) } - return nil, nil + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -150,7 +150,7 @@ func (cf *TaskCLFilter) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理, 复用cl-handler的逻辑 -func (cf *TaskCLFilter) PostExecute(r *dcSDK.BKDistResult) error { +func (cf *TaskCLFilter) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { return cf.postExecute(r) } @@ -168,8 +168,8 @@ func (cf *TaskCLFilter) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (cf *TaskCLFilter) LocalExecute(command []string) (int, error) { - return 0, nil +func (cf *TaskCLFilter) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // FinalExecute 清理临时文件 @@ -188,7 +188,7 @@ func (cf *TaskCLFilter) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { }, nil } -func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("cf: start pre execute for: %v", command) // debugRecordFileName(fmt.Sprintf("cl: start pre execute for: %v", command)) @@ -197,7 +197,7 @@ func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, erro dependFile, args, err := ensureCompiler(command) if err != nil { blog.Errorf("cf: pre execute ensure compiler failed %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } cf.dependentFile = dependFile @@ -210,24 +210,25 @@ func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, erro return cf.clhandle.PreExecute(args) } -func (cf *TaskCLFilter) postExecute(r *dcSDK.BKDistResult) error { +func (cf *TaskCLFilter) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("cf: start post execute for: %v", cf.originArgs) - err := cf.clhandle.PostExecute(r) - if err != nil { - return err + bkerr := cf.clhandle.PostExecute(r) + if bkerr.Error != nil { + return bkerr } // save include to txt file blog.Debugf("cf: ready parse ouput [%s] for: %v", r.Results[0].OutputMessage, cf.originArgs) output, err := cf.parseOutput(string(r.Results[0].OutputMessage)) if err != nil { - return err + blog.Warnf("cf: parse output(%s) with error:%v", r.Results[0].OutputMessage, err) + return dcType.ErrorUnknown } r.Results[0].OutputMessage = []byte(output) blog.Debugf("cf: after parse ouput [%s] for: %v", r.Results[0].OutputMessage, cf.originArgs) - return nil + return dcType.ErrorNone } func (cf *TaskCLFilter) parseOutput(s string) (string, error) { diff --git a/src/backend/booster/bk_dist/handler/ue4/handler.go b/src/backend/booster/bk_dist/handler/ue4/handler.go index 2d813a4a..9bd3ddd0 100644 --- a/src/backend/booster/bk_dist/handler/ue4/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/handler.go @@ -10,7 +10,6 @@ package ue4 import ( - "fmt" "os" "strings" @@ -175,9 +174,10 @@ func (u *UE4) PreLockWeight(command []string) int32 { } // PreExecute 预处理, 根据不同的command来确定不同的子场景 -func (u *UE4) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (u *UE4) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { if command == nil || len(command) == 0 { - return nil, fmt.Errorf("command is nil") + blog.Warnf("command is nil") + return nil, dcType.ErrorUnknown } u.initInnerHandle(command) @@ -189,7 +189,8 @@ func (u *UE4) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { return u.innerhandler.PreExecute(command) } - return nil, fmt.Errorf("not support for command %s", command[0]) + blog.Warnf("not support for command %s", command[0]) + return nil, dcType.ErrorUnknown } // PreExecute 预处理, 根据不同的command来确定不同的子场景 @@ -269,12 +270,12 @@ func (u *UE4) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (u *UE4) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { +func (u *UE4) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { if u.innerhandler != nil { return u.innerhandler.OnRemoteFail(command) } - return nil, nil + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -287,12 +288,13 @@ func (u *UE4) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理 -func (u *UE4) PostExecute(r *dcSDK.BKDistResult) error { +func (u *UE4) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { if u.innerhandler != nil { return u.innerhandler.PostExecute(r) } - return fmt.Errorf("not support") + blog.Warnf("innerhandler is nil when ready post execute") + return dcType.ErrorUnknown } // LocalExecuteNeed no need @@ -321,12 +323,12 @@ func (u *UE4) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (u *UE4) LocalExecute(command []string) (int, error) { +func (u *UE4) LocalExecute(command []string) dcType.BKDistCommonError { if u.innerhandler != nil { return u.innerhandler.LocalExecute(command) } - return 0, nil + return dcType.ErrorNone } // FinalExecute 清理临时文件 diff --git a/src/backend/booster/bk_dist/handler/ue4/lib/handler.go b/src/backend/booster/bk_dist/handler/ue4/lib/handler.go index 09ec8178..0095b031 100644 --- a/src/backend/booster/bk_dist/handler/ue4/lib/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/lib/handler.go @@ -95,7 +95,7 @@ func (l *TaskLib) PreLockWeight(command []string) int32 { } // PreExecute 预处理 -func (l *TaskLib) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (l *TaskLib) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return l.preExecute(command) } @@ -115,8 +115,8 @@ func (l *TaskLib) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (l *TaskLib) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (l *TaskLib) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -125,7 +125,7 @@ func (l *TaskLib) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理 -func (l *TaskLib) PostExecute(r *dcSDK.BKDistResult) error { +func (l *TaskLib) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { return l.postExecute(r) } @@ -140,8 +140,8 @@ func (l *TaskLib) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (l *TaskLib) LocalExecute(command []string) (int, error) { - return 0, nil +func (l *TaskLib) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // FinalExecute no need @@ -163,19 +163,19 @@ func (l *TaskLib) workerSupportAbsPath() bool { return true } -func (l *TaskLib) preExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (l *TaskLib) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("lib: start pre execute for: %v", command) if !l.workerSupportAbsPath() { blog.Infof("lib: remote worker do not support absolute path") - return nil, fmt.Errorf("remote worker do not support absolute path") + return nil, dcType.ErrorUnknown } l.originArgs = command responseFile, args, err := ensureCompiler(command) if err != nil { blog.Errorf("lib: pre execute ensure compiler failed %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } if responseFile != "" && !filepath.IsAbs(responseFile) { @@ -187,7 +187,7 @@ func (l *TaskLib) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err = l.scan(args); err != nil { blog.Errorf("lib: scan args[%v] failed : %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } // add response file as input @@ -201,7 +201,7 @@ func (l *TaskLib) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if !existed { err := fmt.Errorf("input pre file %s not existed", v) blog.Errorf("%v", err) - return nil, err + return nil, dcType.ErrorUnknown } // generate the input files for pre-process file @@ -239,21 +239,22 @@ func (l *TaskLib) preExecute(command []string) (*dcSDK.BKDistCommand, error) { }, }, CustomSave: true, - }, nil + }, dcType.ErrorNone } -func (l *TaskLib) postExecute(r *dcSDK.BKDistResult) error { +func (l *TaskLib) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("lib: start post execute for: %v", l.originArgs) if r == nil || len(r.Results) == 0 { - return ErrorInvalidParam + blog.Warnf("lib: parameter is invalid") + return dcType.ErrorUnknown } if len(r.Results[0].ResultFiles) > 0 { for _, f := range r.Results[0].ResultFiles { if f.Buffer != nil { if err := saveResultFile(&f); err != nil { - blog.Errorf("lib: failed to save file [%s]", f.FilePath) - return err + blog.Errorf("lib: failed to save file [%s] with error:%v", f.FilePath, err) + return dcType.ErrorUnknown } } } @@ -261,13 +262,15 @@ func (l *TaskLib) postExecute(r *dcSDK.BKDistResult) error { if r.Results[0].RetCode == 0 { blog.Infof("lib: success done post execute for: %v", l.originArgs) - return nil + return dcType.ErrorNone } - return fmt.Errorf("lib: failed to remote execute, retcode %d, error message:%s, output message:%s", + blog.Warnf("lib: failed to remote execute, retcode %d, error message:%s, output message:%s", r.Results[0].RetCode, r.Results[0].ErrorMessage, r.Results[0].OutputMessage) + + return dcType.ErrorUnknown } func (l *TaskLib) scan(args []string) error { diff --git a/src/backend/booster/bk_dist/handler/ue4/link/handler.go b/src/backend/booster/bk_dist/handler/ue4/link/handler.go index ea8de397..b037c392 100644 --- a/src/backend/booster/bk_dist/handler/ue4/link/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/link/handler.go @@ -111,7 +111,7 @@ func (l *TaskLink) PreLockWeight(command []string) int32 { } // PreExecute 预处理 -func (l *TaskLink) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (l *TaskLink) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return l.preExecute(command) } @@ -131,8 +131,8 @@ func (l *TaskLink) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (l *TaskLink) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (l *TaskLink) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -141,7 +141,7 @@ func (l *TaskLink) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理 -func (l *TaskLink) PostExecute(r *dcSDK.BKDistResult) error { +func (l *TaskLink) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { return l.postExecute(r) } @@ -156,8 +156,8 @@ func (l *TaskLink) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (l *TaskLink) LocalExecute(command []string) (int, error) { - return 0, nil +func (l *TaskLink) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // FinalExecute no need @@ -179,25 +179,25 @@ func (l *TaskLink) workerSupportAbsPath() bool { return true } -func (l *TaskLink) preExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (l *TaskLink) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("link: start pre execute for: %v", command) if !l.workerSupportAbsPath() { blog.Infof("link: remote worker do not support absolute path") - return nil, fmt.Errorf("remote worker do not support absolute path") + return nil, dcType.ErrorUnknown } l.originArgs = command responseFile, args, err := ensureCompiler(command) if err != nil { blog.Errorf("link: pre execute ensure compiler failed %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } for _, v := range ForceLocalFileKeys { if strings.Contains(responseFile, v) { blog.Errorf("link: pre execute found response %s is in force local list, do not deal now", responseFile) - return nil, fmt.Errorf("response file %s is in force local list", responseFile) + return nil, dcType.ErrorPreForceLocal } } @@ -210,7 +210,7 @@ func (l *TaskLink) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if err = l.scan(args); err != nil { blog.Warnf("link: scan command[%v] with error : %v", command, err) - return nil, err + return nil, dcType.ErrorUnknown } // add response file as input @@ -224,7 +224,7 @@ func (l *TaskLink) preExecute(command []string) (*dcSDK.BKDistCommand, error) { if !existed { err := fmt.Errorf("input file %s not existed", v) blog.Errorf("%v", err) - return nil, err + return nil, dcType.ErrorUnknown } // generate the input files for pre-process file @@ -369,21 +369,22 @@ func (l *TaskLink) preExecute(command []string) (*dcSDK.BKDistCommand, error) { blog.Debugf("link: after pre,full command[%v]", req) - return &req, nil + return &req, dcType.ErrorNone } -func (l *TaskLink) postExecute(r *dcSDK.BKDistResult) error { +func (l *TaskLink) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("link: start post execute for: %v", l.originArgs) if r == nil || len(r.Results) == 0 { - return ErrorInvalidParam + blog.Warnf("link: parameter is invalid") + return dcType.ErrorUnknown } if len(r.Results[0].ResultFiles) > 0 { for _, f := range r.Results[0].ResultFiles { if f.Buffer != nil { if err := saveResultFile(&f); err != nil { - blog.Errorf("link: failed to save file [%s]", f.FilePath) - return err + blog.Errorf("link: failed to save file [%s] with error:%v", f.FilePath, err) + return dcType.ErrorUnknown } } } @@ -391,13 +392,15 @@ func (l *TaskLink) postExecute(r *dcSDK.BKDistResult) error { if r.Results[0].RetCode == 0 { blog.Infof("link: success done post execute for: %v", l.originArgs) - return nil + return dcType.ErrorNone } - return fmt.Errorf("link: failed to remote execute, retcode %d, error message:%s, output message:%s", + blog.Warnf("link: failed to remote execute, retcode %d, error message:%s, output message:%s", r.Results[0].RetCode, r.Results[0].ErrorMessage, r.Results[0].OutputMessage) + + return dcType.ErrorUnknown } func (l *TaskLink) scan(args []string) error { diff --git a/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go b/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go index fabc207c..5798804a 100644 --- a/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/linkfilter/handler.go @@ -99,7 +99,7 @@ func (lf *TaskLinkFilter) PreLockWeight(command []string) int32 { } // PreExecute 预处理, 复用cl-handler的逻辑 -func (lf *TaskLinkFilter) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (lf *TaskLinkFilter) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { return lf.preExecute(command) } @@ -119,8 +119,8 @@ func (lf *TaskLinkFilter) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (lf *TaskLinkFilter) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (lf *TaskLinkFilter) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -132,7 +132,7 @@ func (lf *TaskLinkFilter) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理, 复用cl-handler的逻辑 -func (lf *TaskLinkFilter) PostExecute(r *dcSDK.BKDistResult) error { +func (lf *TaskLinkFilter) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { return lf.postExecute(r) } @@ -150,8 +150,8 @@ func (lf *TaskLinkFilter) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (cf *TaskLinkFilter) LocalExecute(command []string) (int, error) { - return 0, nil +func (cf *TaskLinkFilter) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // FinalExecute 清理临时文件 @@ -166,18 +166,19 @@ func (lf *TaskLinkFilter) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { return nil, nil } -func (lf *TaskLinkFilter) preExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (lf *TaskLinkFilter) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("lf: start pre execute for: %v", command) if lf.handle == nil { - return nil, ErrorNilInnerHandle + blog.Warnf("lf: inner handle is nil") + return nil, dcType.ErrorUnknown } lf.originArgs = command args, err := ensureCompiler(command) if err != nil { blog.Errorf("lf: pre execute ensure compiler failed %v: %v", args, err) - return nil, err + return nil, dcType.ErrorUnknown } lf.linkArgs = args @@ -188,11 +189,12 @@ func (lf *TaskLinkFilter) preExecute(command []string) (*dcSDK.BKDistCommand, er return lf.handle.PreExecute(args) } -func (lf *TaskLinkFilter) postExecute(r *dcSDK.BKDistResult) error { +func (lf *TaskLinkFilter) postExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("lf: start post execute for: %v", lf.originArgs) if lf.handle == nil { - return ErrorNilInnerHandle + blog.Warnf("lf: inner handle is nil") + return dcType.ErrorUnknown } return lf.handle.PostExecute(r) diff --git a/src/backend/booster/bk_dist/handler/ue4/shader/handler.go b/src/backend/booster/bk_dist/handler/ue4/shader/handler.go index 25a09d25..0a1b912b 100644 --- a/src/backend/booster/bk_dist/handler/ue4/shader/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/shader/handler.go @@ -10,7 +10,6 @@ package shader import ( - "fmt" "os" "path/filepath" "runtime" @@ -103,7 +102,7 @@ func (u *UE4Shader) isNewShader() bool { } // PreExecute 预处理 -func (u *UE4Shader) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (u *UE4Shader) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Infof("shader: ready pre execute with command[%v]", command) // to support ue 5.3 macos @@ -112,7 +111,8 @@ func (u *UE4Shader) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { } if len(command) < 6 { - return nil, fmt.Errorf("shader: invalid command") + blog.Warnf("shader: invalid command") + return nil, dcType.ErrorUnknown } filedir, _ := filepath.Abs(command[1]) @@ -140,7 +140,8 @@ func (u *UE4Shader) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { info := dcFile.Stat(inputFile) existed, fileSize, modifyTime, fileMode := info.Batch() if !existed { - return nil, fmt.Errorf("shader: input file %s not exist with error:%v", inputFile, info.Error()) + blog.Warnf("shader: input file %s not exist with error:%v", inputFile, info.Error()) + return nil, dcType.ErrorUnknown } inputfiles := []dcSDK.FileDesc{{ @@ -185,14 +186,15 @@ func (u *UE4Shader) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { }, }, CustomSave: true, - }, nil + }, dcType.ErrorNone } -func (u *UE4Shader) PreExecuteNew(command []string) (*dcSDK.BKDistCommand, error) { +func (u *UE4Shader) PreExecuteNew(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { blog.Debugf("shader: ready pre execute new with command[%v]", command) if len(command) < 6 { - return nil, fmt.Errorf("shader: invalid command") + blog.Warnf("shader: invalid command") + return nil, dcType.ErrorUnknown } // 注意:command[1]的路径后面可能有/,这个不能去掉,否则shader会编译失败 @@ -227,7 +229,8 @@ func (u *UE4Shader) PreExecuteNew(command []string) (*dcSDK.BKDistCommand, error info := dcFile.Stat(inputFile) existed, fileSize, modifyTime, fileMode := info.Batch() if !existed { - return nil, fmt.Errorf("shader: input file %s not exist with error:%v", inputFile, info.Error()) + blog.Warnf("shader: input file %s not exist with error:%v", inputFile, info.Error()) + return nil, dcType.ErrorUnknown } inputfiles := []dcSDK.FileDesc{{ @@ -273,7 +276,7 @@ func (u *UE4Shader) PreExecuteNew(command []string) (*dcSDK.BKDistCommand, error }, }, CustomSave: true, - }, nil + }, dcType.ErrorNone } // NeedRemoteResource check whether this command need remote resource @@ -292,8 +295,8 @@ func (u *UE4Shader) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (u *UE4Shader) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (u *UE4Shader) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -302,38 +305,45 @@ func (u *UE4Shader) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理 -func (u *UE4Shader) PostExecute(r *dcSDK.BKDistResult) error { +func (u *UE4Shader) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { if r == nil || len(r.Results) == 0 { - return fmt.Errorf("shader: result data is invalid") + blog.Warnf("shader: result data is invalid") + return dcType.ErrorUnknown } result := r.Results[0] if result.RetCode != 0 { - return fmt.Errorf("shader: failed to remote execute, retcode %d, "+ + blog.Warnf("shader: failed to remote execute, retcode %d, "+ "error message:[%s], output message:[%s]", result.RetCode, result.ErrorMessage, result.OutputMessage) + + return dcType.ErrorUnknown } if len(result.ResultFiles) == 0 { - return fmt.Errorf("shader: not found result file, retcode %d, error message:[%s], output message:[%s]", + blog.Warnf("shader: not found result file, retcode %d, error message:[%s], output message:[%s]", result.RetCode, result.ErrorMessage, result.OutputMessage) + + return dcType.ErrorUnknown } err := checkAndsaveResultFile(&result.ResultFiles[0]) if err != nil { blog.Infof("shader: failed to check and save shader result file[%s],error:[%v]", result.ResultFiles[0].FilePath, err) + + return dcType.ErrorUnknown } // move result temp to real blog.Infof("shader: ready rename file from [%s] to [%s]", u.outputTempFile, u.outputRealFile) _ = os.Rename(u.outputTempFile, u.outputRealFile) - return err + return dcType.ErrorNone } // LocalExecuteNeed no need @@ -372,8 +382,8 @@ func (u *UE4Shader) LocalLockWeight(command []string) int32 { } // LocalExecute no need -func (u *UE4Shader) LocalExecute(command []string) (int, error) { - return 0, nil +func (u *UE4Shader) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // FinalExecute no need diff --git a/src/backend/booster/bk_dist/handler/winclangcl/handler.go b/src/backend/booster/bk_dist/handler/winclangcl/handler.go index a51d46d4..201c8255 100644 --- a/src/backend/booster/bk_dist/handler/winclangcl/handler.go +++ b/src/backend/booster/bk_dist/handler/winclangcl/handler.go @@ -83,14 +83,14 @@ func (cc *WinClangCl) PreLockWeight(command []string) int32 { } // PreExecute 预处理 -func (cc *WinClangCl) PreExecute(command []string) (*dcSDK.BKDistCommand, error) { +func (cc *WinClangCl) PreExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { commandline, err := cmd.Parse(command) if err != nil { - return nil, err + return nil, dcType.ErrorUnknown } err = cc.preprocess(commandline) if err != nil { - return nil, err + return nil, dcType.ErrorUnknown } serverSideArgs := commandline.RenderToServerSide(cc.preprocessedFile) @@ -105,7 +105,7 @@ func (cc *WinClangCl) PreExecute(command []string) (*dcSDK.BKDistCommand, error) ResultFiles: []string{commandline.Obj}, }, }, - }, nil + }, dcType.ErrorNone } // LocalExecuteNeed 无需自定义本地处理 @@ -119,8 +119,8 @@ func (cc *WinClangCl) LocalLockWeight(command []string) int32 { } // LocalExecute 无需自定义本地处理 -func (cc *WinClangCl) LocalExecute(command []string) (int, error) { - return 0, nil +func (cc *WinClangCl) LocalExecute(command []string) dcType.BKDistCommonError { + return dcType.ErrorNone } // NeedRemoteResource check whether this command need remote resource @@ -139,8 +139,8 @@ func (cc *WinClangCl) NeedRetryOnRemoteFail(command []string) bool { } // OnRemoteFail give chance to try other way if failed to remote execute -func (cc *WinClangCl) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { - return nil, nil +func (cc *WinClangCl) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) { + return nil, dcType.ErrorNone } // PostLockWeight decide post-execute lock weight, default 1 @@ -149,21 +149,24 @@ func (cc *WinClangCl) PostLockWeight(result *dcSDK.BKDistResult) int32 { } // PostExecute 后置处理, 判断远程执行的结果是否正确 -func (cc *WinClangCl) PostExecute(r *dcSDK.BKDistResult) error { +func (cc *WinClangCl) PostExecute(r *dcSDK.BKDistResult) dcType.BKDistCommonError { blog.Infof("cc: [%s] start post execute", cc.tag) if r == nil || len(r.Results) == 0 { - return fmt.Errorf("cc: remote execute error") + blog.Warnf("cc: remote execute error") + return dcType.ErrorUnknown } if r.Results[0].RetCode == 0 { blog.Infof("cc: [%s] success done post execute", cc.tag) - return nil + return dcType.ErrorNone } - return fmt.Errorf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s", + blog.Warnf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s", cc.tag, r.Results[0].RetCode, r.Results[0].ErrorMessage, r.Results[0].OutputMessage) + + return dcType.ErrorUnknown } // FinalExecute 清理临时文件 @@ -190,7 +193,7 @@ func (cc *WinClangCl) GetPreloadConfig(config dcType.BoosterConfig) (*dcSDK.Prel return nil, nil } -//编译过程的预处理 +// 编译过程的预处理 func (cc *WinClangCl) preprocess(command cmd.Commandline) error { sandbox := cc.sandbox.Fork() tmpfile, err := ioutil.TempFile("", "bk_boost_*."+command.FileType)