diff --git a/src/backend/booster/bk_dist/common/sdk/toolchain.go b/src/backend/booster/bk_dist/common/sdk/toolchain.go index 6cedd19a..425edc94 100644 --- a/src/backend/booster/bk_dist/common/sdk/toolchain.go +++ b/src/backend/booster/bk_dist/common/sdk/toolchain.go @@ -11,6 +11,8 @@ package sdk import ( "fmt" + "os" + "path/filepath" "strings" dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" @@ -95,50 +97,157 @@ func ResolveToolchainEnvValue(value string) (map[string]string, error) { return outmap, nil } +func checkAndAdd(i *dcFile.Info, remotepath string, files *[]FileDesc) error { + f := FileDesc{ + FilePath: i.Path(), + Compresstype: protocol.CompressLZ4, + FileSize: i.Size(), + Lastmodifytime: i.ModifyTime64(), + Md5: "", + Targetrelativepath: remotepath, + Filemode: i.Mode32(), + LinkTarget: i.LinkTarget, + } + + if i.LinkTarget == "" { + *files = append(*files, f) + return nil + } + + // 检查链接是否存在循环 + for _, v := range *files { + if v.FilePath == i.Path() { + return fmt.Errorf("found loop link file:%s", v.FilePath) + } + } + + *files = append(*files, f) + return nil +} + +// 得到所有关联文件；如果是链接，则递归搜索，直到找到非链接为止 +// 如果发现链接循环，则报错 +func getRecursiveFiles(f string, remotepath string, files *[]FileDesc) error { + i := dcFile.Lstat(f) + if !i.Exist() { + err := fmt.Errorf("file %s not existed", f) + blog.Errorf("%v", err) + return err + } + + // 链接，需要递归 + if i.Basic().Mode()&os.ModeSymlink != 0 { + originFile, err := os.Readlink(f) + if err == nil { + if !filepath.IsAbs(originFile) { + originFile, err = filepath.Abs(filepath.Join(filepath.Dir(f), originFile)) + if err == nil { + i.LinkTarget = originFile + blog.Infof("toolchain: symlink %s to %s", f, originFile) + } else { + blog.Infof("toolchain: symlink %s origin %s, got abs path error:%s", + f, originFile, err) + return err + } + } else { + i.LinkTarget = originFile + blog.Infof("toolchain: symlink %s to %s", f, originFile) + } + + err = checkAndAdd(i, remotepath, files) + if err != nil { + return err + } + + // 递归查找 + return getRecursiveFiles(originFile, filepath.Dir(originFile), files) + } else { + blog.Infof("toolchain: symlink %s Readlink error:%s", f, err) + return err + } + } + + return checkAndAdd(i, remotepath, files) +} + +// 得到所有关联文件 +func getAssociatedFiles(f string, remotepath string) (*[]FileDesc, error) { + files := make([]FileDesc, 0, 0) + err := getRecursiveFiles(f, remotepath, &files) + + return &files, err +} + // ToFileDesc parse toolchains to file targets func (t *Toolchain) ToFileDesc() ([]FileDesc, error) { if t == nil { return nil, fmt.Errorf("tool chain is nil") } + // TODO : 将链接展开，直到得到所有相关文件，比如 a->b,b->c，则需要将a/b/c都包含进来 toolfiles := make([]FileDesc, 0, 0) for _, v := range t.Toolchains { - existed, fileSize, modifyTime, fileMode := dcFile.Stat(v.ToolLocalFullPath).Batch() - if !existed { - err := fmt.Errorf("tool chain file %s not existed", v.ToolLocalFullPath) - blog.Errorf("%v", err) + // existed, fileSize, modifyTime, fileMode := dcFile.Stat(v.ToolLocalFullPath).Batch() + // if !existed { + // err := fmt.Errorf("tool chain file %s not existed", v.ToolLocalFullPath) + // blog.Errorf("%v", err) + // return nil, err + // } + + // toolfiles = append(toolfiles, FileDesc{ + // FilePath: v.ToolLocalFullPath, + // Compresstype: protocol.CompressLZ4, + // FileSize: fileSize, + // Lastmodifytime: modifyTime, + // Md5: "", + // Targetrelativepath: v.ToolRemoteRelativePath, + // Filemode: fileMode, + // }) + files, err := getAssociatedFiles(v.ToolLocalFullPath, v.ToolRemoteRelativePath) + if err != nil { return nil, err } - - toolfiles = append(toolfiles, FileDesc{ - FilePath: v.ToolLocalFullPath, - Compresstype: protocol.CompressLZ4, - FileSize: fileSize, - Lastmodifytime: modifyTime, - Md5: "", - Targetrelativepath: v.ToolRemoteRelativePath, - Filemode: fileMode, - }) + // 倒序添加，保证创建链接成功 + size := len(*files) + if size > 0 { + for i := size - 1; i >= 0; i-- { + toolfiles = append(toolfiles, (*files)[i]) + } + } for _, f := range v.Files { - existed, fileSize, modifyTime, fileMode = dcFile.Stat(f.LocalFullPath).Batch() - if !existed { - err := fmt.Errorf("tool chain file %s not existed", f.LocalFullPath) - blog.Errorf("%v", err) + // existed, fileSize, modifyTime, fileMode = dcFile.Stat(f.LocalFullPath).Batch() + // if !existed { + // err := fmt.Errorf("tool chain file %s not existed", f.LocalFullPath) + // blog.Errorf("%v", err) + // return nil, err + // } + + // toolfiles = append(toolfiles, FileDesc{ + // FilePath: f.LocalFullPath, + // Compresstype: protocol.CompressLZ4, + // FileSize: fileSize, + // Lastmodifytime: modifyTime, + // Md5: "", + // Targetrelativepath: f.RemoteRelativePath, + // Filemode: fileMode, + // }) + + files, err := getAssociatedFiles(f.LocalFullPath, f.RemoteRelativePath) + if err != nil { return nil, err } - - toolfiles = append(toolfiles, FileDesc{ - FilePath: f.LocalFullPath, - Compresstype: protocol.CompressLZ4, - FileSize: fileSize, - Lastmodifytime: modifyTime, - Md5: "", - Targetrelativepath: f.RemoteRelativePath, - Filemode: fileMode, - }) + // 倒序添加，保证创建链接成功 + size := len(*files) + if size > 0 { + for i := size - 1; i >= 0; i-- { + toolfiles = append(toolfiles, (*files)[i]) + } + } } } + blog.Infof("toolchain: get all files:%v", toolfiles) + return toolfiles, nil } diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go index 9cb47916..9db6a8fb 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go @@ -527,6 +527,9 @@ func (m *Mgr) ExecuteTask(req *types.RemoteTaskExecuteRequest) (*types.RemoteTas dcSDK.StatsTimeNow(&req.Stats.RemoteWorkStartTime) m.work.Basic().UpdateJobStats(req.Stats) + blog.Infof("remote: try to real execute remote task for work(%s) from pid(%d) with timeout(%d) after send files", + m.work.ID(), req.Pid, req.IOTimeout) + var result *dcSDK.BKDistResult if m.conf.LongTCP { if !req.Req.CustomSave { @@ -653,9 +656,9 @@ func (m *Mgr) ensureFiles( wg := make(chan error, len(fileDetails)+1) count := 0 r := make([]string, 0, 10) - cleaner := make([]dcSDK.FileDesc, 0, 10) + // cleaner := make([]dcSDK.FileDesc, 0, 10) corkFiles := make(map[string]*[]*corkFile, 0) - allServerCorkFiles := make(map[string]*[]*corkFile, 0) + // allServerCorkFiles := make(map[string]*[]*corkFile, 0) filesNum := len(fileDetails) for _, fd := range fileDetails { blog.Debugf("remote: debug try to ensure file %+v", *fd) @@ -687,13 +690,28 @@ func (m *Mgr) ensureFiles( continue case dcSDK.FilterRuleHandleAllDistribution: - cleaner = append(cleaner, f) + // cleaner = append(cleaner, f) if err = m.fileMessageBank.ensure(sender, sandbox); err != nil { return nil, err } // 该文件需要被分发到所有的机器上 servers = m.work.Resource().GetHosts() + if len(servers) > 0 { + for _, s := range servers { + existed := false + for _, s1 := range fd.Servers { + if s.Equal(s1) { + existed = true + break + } + } + + if !existed { + fd.Servers = append(fd.Servers, s) + } + } + } } r = append(r, f.Targetrelativepath) @@ -735,39 +753,40 @@ func (m *Mgr) ensureFiles( } } - // 分发额外的内容 - for _, s := range servers { - if !m.conf.SendCork { - go func(host *dcProtocol.Host, req *dcSDK.BKDistFileSender) { - t := time.Now().Local() - _ = m.ensureSingleFile(handler, host, req, sandbox) - d := time.Now().Local().Sub(t) - if d > 200*time.Millisecond { - blog.Debugf("remote: single file cost time for work(%s) from pid(%d) to server(%s): %s, %s", - m.work.ID(), pid, host.Server, d.String(), req.Files[0].FilePath) - } - }(s, sender) - } else { - // for send cork - cf := &corkFile{ - handler: handler, - host: s, - sandbox: sandbox, - file: &f, - resultchan: nil, - } - l, ok := allServerCorkFiles[s.Server] - if !ok { - // 预先分配好队列，避免频繁内存分配 - // newl := []*corkFile{cf} - newl := make([]*corkFile, 0, filesNum) - newl = append(newl, cf) - allServerCorkFiles[s.Server] = &newl - } else { - *l = append(*l, cf) - } - } - } + // // 分发额外的内容 + // for _, s := range servers { + // count++ + // if !m.conf.SendCork { + // go func(err chan<- error, host *dcProtocol.Host, req *dcSDK.BKDistFileSender) { + // t := time.Now().Local() + // err <- m.ensureSingleFile(handler, host, req, sandbox) + // d := time.Now().Local().Sub(t) + // if d > 200*time.Millisecond { + // blog.Debugf("remote: single file cost time for work(%s) from pid(%d) to server(%s): %s, %s", + // m.work.ID(), pid, host.Server, d.String(), req.Files[0].FilePath) + // } + // }(wg, s, sender) + // } else { + // // for send cork + // cf := &corkFile{ + // handler: handler, + // host: s, + // sandbox: sandbox, + // file: &f, + // resultchan: nil, + // } + // l, ok := allServerCorkFiles[s.Server] + // if !ok { + // // 预先分配好队列，避免频繁内存分配 + // // newl := []*corkFile{cf} + // newl := make([]*corkFile, 0, filesNum) + // newl = append(newl, cf) + // allServerCorkFiles[s.Server] = &newl + // } else { + // *l = append(*l, cf) + // } + // } + // } } if m.conf.SendCork { @@ -817,57 +836,71 @@ func (m *Mgr) ensureFiles( m.sendCorkChan <- true } - // same with corkFiles, but do not notify wg - for server, fs := range allServerCorkFiles { - totalFileNum := len(*fs) - descs := make([]*dcSDK.FileDesc, 0, totalFileNum) - for _, v := range *fs { - descs = append(descs, v.file) - } - results := m.checkOrLockCorkFiles(server, descs) - needSendCorkFiles := make([]*corkFile, 0, totalFileNum) - for i, v := range results { - if v.match { - // 已发送完成的不启动协程了 - if v.info.SendStatus == types.FileSendSucceed { - continue - } else if v.info.SendStatus == types.FileSendFailed { - continue - } - } else { - // 不在缓存，意味着之前没有发送过 - (*fs)[i].resultchan = make(chan corkFileResult, 1) - needSendCorkFiles = append(needSendCorkFiles, (*fs)[i]) - } - - // 启动协程跟踪未发送完成的文件 - c := (*fs)[i] - go func(c *corkFile, r matchResult) { - _ = m.ensureSingleCorkFile(c, r) - }(c, v) - } - - blog.Debugf("total %d cork files, need send %d files", totalFileNum, len(needSendCorkFiles)) - // append to cork files queue - _ = m.appendCorkFiles(server, needSendCorkFiles) - - // notify send - m.sendCorkChan <- true - } + // // same with corkFiles, but do not notify wg + // for server, fs := range allServerCorkFiles { + // totalFileNum := len(*fs) + // descs := make([]*dcSDK.FileDesc, 0, totalFileNum) + // for _, v := range *fs { + // descs = append(descs, v.file) + // } + // results := m.checkOrLockCorkFiles(server, descs) + // needSendCorkFiles := make([]*corkFile, 0, totalFileNum) + // for i, v := range results { + // if v.match { + // // 已发送完成的不启动协程了 + // if v.info.SendStatus == types.FileSendSucceed { + // wg <- nil + // continue + // } else if v.info.SendStatus == types.FileSendFailed { + // wg <- nil + // continue + // } + // } else { + // // 不在缓存，意味着之前没有发送过 + // (*fs)[i].resultchan = make(chan corkFileResult, 1) + // needSendCorkFiles = append(needSendCorkFiles, (*fs)[i]) + // } + + // // 启动协程跟踪未发送完成的文件 + // c := (*fs)[i] + // go func(err chan<- error, c *corkFile, r matchResult) { + // err <- m.ensureSingleCorkFile(c, r) + // }(wg, c, v) + // } + + // blog.Debugf("total %d cork files, need send %d files", totalFileNum, len(needSendCorkFiles)) + // // append to cork files queue + // _ = m.appendCorkFiles(server, needSendCorkFiles) + + // // notify send + // m.sendCorkChan <- true + // } } for i := 0; i < count; i++ { if err = <-wg; err != nil { blog.Warnf("remote: failed to ensure multi %d files for work(%s) from pid(%d) to server with err:%v", count, m.work.ID(), pid, err) + + // 异常情况下启动一个协程将消息收完，避免发送协程阻塞 + i++ + if i < count { + go func(i, count int, c <-chan error) { + for ; i < count; i++ { + _ = <-c + } + }(i, count, wg) + } + return nil, err } } - blog.Debugf("remote: success to ensure multi %d files for work(%s) from pid(%d) to server", + blog.Infof("remote: success to ensure multi %d files for work(%s) from pid(%d) to server", count, m.work.ID(), pid) - for _, f := range cleaner { - go m.fileMessageBank.clean(f) - } + + // for _, f := range cleaner { + // go m.fileMessageBank.clean(f) + // } return r, nil } @@ -1592,6 +1625,7 @@ func (m *Mgr) getToolChainFromExecuteRequest(req *types.RemoteTaskExecuteRequest blog.Infof("remote: ready get normal toolchain files") toolchainfiles, timestamp, err := m.work.Basic().GetToolChainFiles(c.ExeToolChainKey) if err == nil && len(toolchainfiles) > 0 { + blog.Infof("remote: got toolchain files:%v", toolchainfiles) fd = append(fd, &types.FileCollectionInfo{ UniqID: c.ExeToolChainKey, Files: toolchainfiles, diff --git a/src/backend/booster/bk_dist/handler/cc/booster.go b/src/backend/booster/bk_dist/handler/cc/booster.go index 85a46b67..f52a8990 100644 --- a/src/backend/booster/bk_dist/handler/cc/booster.go +++ b/src/backend/booster/bk_dist/handler/cc/booster.go @@ -81,6 +81,15 @@ func (cc *TaskCC) appendCcache(config dcType.BoosterConfig) error { return nil } +func (cc *TaskCC) appendPump(config dcType.BoosterConfig) error { + if config.Works.PumpCache { + bazelActionConstOptions = append(bazelActionConstOptions, env.GetEnvKey(env.KeyExecutorPumpCache)) + // 这个是p2p里面需要判断的，先忽略 + // bazelActionConstOptions = append(bazelActionConstOptions, env.GetEnvKey(env.KeyWorkerSupportAbsPath)） + } + return nil +} + // ProjectExtraData describe the extra data store in project // ccache_enable and ccache_enabled are both to control ccache usage, if one of them is true, then ccache enabled. type ProjectExtraData struct { @@ -157,6 +166,7 @@ func (cc *TaskCC) RenderArgs(config dcType.BoosterConfig, originArgs string) str appendPreload() cc.appendCcache(config) + cc.appendPump(config) if config.Works.BazelPlus || config.Works.Bazel4Plus || config.Works.BazelNoLauncher { additions := make([]string, 0, 10) diff --git a/src/backend/booster/bk_dist/handler/cc/error.go b/src/backend/booster/bk_dist/handler/cc/error.go index e0f31e81..52d58035 100644 --- a/src/backend/booster/bk_dist/handler/cc/error.go +++ b/src/backend/booster/bk_dist/handler/cc/error.go @@ -31,7 +31,13 @@ var ( ErrorNotSupportSpecs = fmt.Errorf("-specs= must be local") ErrorNotSupportX = fmt.Errorf("-x must be local") ErrorNotSupportDr = fmt.Errorf("-dr must be local") + ErrorNotSupportFsanitize = fmt.Errorf("-fsanitize must be local") ErrorNotSupportConftest = fmt.Errorf("tmp.conftest. must be local") ErrorNotSupportOutputStdout = fmt.Errorf("output with - to stdout, must be local") ErrorNotSupportGch = fmt.Errorf("output with .gch, must be local") + ErrorNoPumpHeadFile = fmt.Errorf("pump head file not exist") + ErrorNoDependFile = fmt.Errorf("depend file not exist") + ErrorInvalidDependFile = fmt.Errorf("depend file invalid") + ErrorNotSupportRemote = fmt.Errorf("not support to remote execute") + ErrorInPumpBlack = fmt.Errorf("in pump black list") ) diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go index 9d089b9e..e9b6ec10 100644 --- a/src/backend/booster/bk_dist/handler/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/cc/handler.go @@ -17,12 +17,17 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "time" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env" + dcEnv "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env" dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol" + dcPump "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/pump" dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" + dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/controller/pkg/manager/analyser" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler" commonUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler/common" @@ -31,6 +36,10 @@ import ( "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/util" ) +const ( + appendEnvKey = "INCLUDE=" +) + var ( DefaultForceLocalCppFileKeys = make([]string, 0, 0) ) @@ -54,6 +63,7 @@ type TaskCC struct { rewriteCrossArgs []string preProcessArgs []string serverSideArgs []string + pumpArgs []string // file names inputFile string @@ -66,9 +76,34 @@ type TaskCC struct { supportDirectives bool + responseFile string + sourcedependfile string + pumpHeadFile string + includeRspFiles []string // 在rsp中通过@指定的其它rsp文件，需要发送到远端 + // 在rsp中-I后面的参数，需要将这些目录全部发送到远端 + // 有特殊场景：编译不需要该路径下的文件，但需要该路径作为跳板，去查找其它相对路径下的头文件（或其它依赖文件） + includePaths []string + // 在rsp中 -include后面的参数，将这些文件发送到远端 + includeFiles []string + + // forcedepend 是我们主动导出依赖文件，showinclude 是编译命令已经指定了导出依赖文件 + forcedepend bool + pumpremote bool + needcopypumpheadfile bool + pumpremotefailed bool + + // 当前是pch/gch的生成，pump模式下我们需要关注并保存其依赖关系 + // 方便在后续pump模式下，发送完整的pch/gch列表 + needresolvepchdepend bool + objectpchfile string + pchFileDesc *dcSDK.FileDesc - ForceLocalCppFileKeys []string + // for /showIncludes + showinclude bool + + ForceLocalResponseFileKeys []string + ForceLocalCppFileKeys []string } // NewTaskCC get a new task-cc handler @@ -146,11 +181,20 @@ func (cc *TaskCC) RemoteRetryTimes() int { // NeedRetryOnRemoteFail check whether need retry on remote fail func (cc *TaskCC) NeedRetryOnRemoteFail(command []string) bool { - return false + return cc.pumpremote } // OnRemoteFail give chance to try other way if failed to remote execute func (cc *TaskCC) OnRemoteFail(command []string) (*dcSDK.BKDistCommand, error) { + blog.Infof("cc: start OnRemoteFail for: %v", command) + + if cc.pumpremote { + blog.Infof("cc: set pumpremotefailed to true now") + cc.pumpremotefailed = true + cc.needcopypumpheadfile = true + cc.pumpremote = false + return cc.preExecute(command) + } return nil, nil } @@ -181,10 +225,555 @@ func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { }, nil } +func uniqArr(arr []string) []string { + newarr := make([]string, 0) + tempMap := make(map[string]bool, len(newarr)) + for _, v := range arr { + if tempMap[v] == false { + tempMap[v] = true + newarr = append(newarr, v) + } + } + + return newarr +} + +func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) { + data, err := os.ReadFile(dependf) + if err != nil { + return nil, err + } + + sep := "\n" + lines := strings.Split(string(data), sep) + uniqlines := uniqArr(lines) + blog.Infof("cc: got %d uniq include file from file: %s", len(uniqlines), dependf) + + return commonUtil.GetFileInfo(uniqlines, false, false, dcPump.SupportPumpLstatByDir(cc.sandbox.Env)) +} + +func (cc *TaskCC) checkFstat(f string, workdir string) (*dcFile.Info, error) { + if !filepath.IsAbs(f) { + f, _ = filepath.Abs(filepath.Join(workdir, f)) + } + fstat := dcFile.Stat(f) + if fstat.Exist() && !fstat.Basic().IsDir() { + return fstat, nil + } + + return nil, nil +} + +func (cc *TaskCC) resolveDependFile(sep, workdir string, includes *[]string) error { + data, err := os.ReadFile(cc.sourcedependfile) + if err != nil { + blog.Warnf("cc: copy pump head failed to read depned file: %s with err:%v", cc.sourcedependfile, err) + return err + } + + lines := strings.Split(string(data), sep) + // includes := []string{} + for _, l := range lines { + l = strings.Trim(l, " \r\n\\") + // TODO : the file path maybe contains space, should support this condition + fields := strings.Split(l, " ") + if len(fields) >= 1 { + // for i, f := range fields { + for index := len(fields) - 1; index >= 0; index-- { + var targetf string + // /xx/xx/aa .cpp /xx/xx/aa .h /xx/xx/aa .o + // 支持文件名后缀前有空格的情况，但没有支持路径中间有空格的，比如 /xx /xx /aa.cpp + // 向前依附到不为空的字符串为止 + if fields[index] == ".cpp" || fields[index] == ".h" || fields[index] == ".o" { + for targetindex := index - 1; targetindex >= 0; targetindex-- { + if len(fields[targetindex]) > 0 { + fields[targetindex] = strings.Trim(fields[targetindex], "\\") + targetf = strings.Join(fields[targetindex:index+1], " ") + index = targetindex + break + } + } + } else if len(fields[index]) > 0 { + targetf = fields[index] + } else { + continue + } + + if strings.HasSuffix(targetf, ".o:") || strings.HasSuffix(targetf, ".gch:") { + continue + } + if !filepath.IsAbs(targetf) { + targetf, _ = filepath.Abs(filepath.Join(workdir, targetf)) + } + + *includes = append(*includes, commonUtil.FormatFilePath(targetf)) + } + } + } + + return nil +} + +func (cc *TaskCC) copyPumpHeadFile(workdir string) error { + blog.Infof("cc: copy pump head file: %s to: %s", cc.sourcedependfile, cc.pumpHeadFile) + + sep := "\n" + includes := []string{} + err := cc.resolveDependFile(sep, workdir, &includes) + if err != nil { + return err + } + + // copy includeRspFiles + if len(cc.includeRspFiles) > 0 { + for _, l := range cc.includeRspFiles { + blog.Infof("cc: ready add rsp file: %s", l) + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + includes = append(includes, commonUtil.FormatFilePath(l)) + } + } + + // copy includePaths + if len(cc.includePaths) > 0 { + for _, l := range cc.includePaths { + blog.Infof("cc: ready add include path: %s", l) + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + includes = append(includes, commonUtil.FormatFilePath(l)) + } + } + + // copy include files + if len(cc.includeFiles) > 0 { + for _, l := range cc.includeFiles { + blog.Infof("cc: ready add include file: %s", l) + if !filepath.IsAbs(l) { + l, _ = filepath.Abs(filepath.Join(workdir, l)) + } + includes = append(includes, commonUtil.FormatFilePath(l)) + } + } + + blog.Infof("cc: copy pump head got %d uniq include file from file: %s", len(includes), cc.sourcedependfile) + + if len(includes) == 0 { + blog.Warnf("cc: depend file: %s is invalid", cc.sourcedependfile) + return ErrorInvalidDependFile + } + + uniqlines := uniqArr(includes) + + // append symlink or symlinked if need + links, _ := getIncludeLinks(cc.sandbox.Env, uniqlines) + if links != nil { + uniqlines = append(uniqlines, links...) + } + + // save to cc.pumpHeadFile + newdata := strings.Join(uniqlines, sep) + err = os.WriteFile(cc.pumpHeadFile, []byte(newdata), os.ModePerm) + if err != nil { + blog.Warnf("cc: copy pump head failed to write file: %s with err:%v", cc.pumpHeadFile, err) + return err + } else { + blog.Infof("cc: copy pump head succeed to write file: %s", cc.pumpHeadFile) + } + + return nil +} + +func (cc *TaskCC) getPumpDir(env *env.Sandbox) (string, error) { + pumpdir := dcPump.PumpCacheDir(env) + if pumpdir == "" { + pumpdir = dcUtil.GetPumpCacheDir() + } + + if !dcFile.Stat(pumpdir).Exist() { + if err := os.MkdirAll(pumpdir, os.ModePerm); err != nil { + return "", err + } + } + + return pumpdir, nil +} + +// search all include files for this compile command +func (cc *TaskCC) Includes(responseFile string, args []string, workdir string, forcefresh bool) ([]*dcFile.Info, error) { + pumpdir, err := cc.getPumpDir(cc.sandbox.Env) + if err != nil { + return nil, err + } + + cc.pumpHeadFile, err = getPumpIncludeFile(pumpdir, "pump_heads", ".txt", args, workdir) + if err != nil { + blog.Errorf("cc: do includes get output file failed: %v", err) + return nil, err + } + + existed, fileSize, _, _ := dcFile.Stat(cc.pumpHeadFile).Batch() + if dcPump.IsPumpCache(cc.sandbox.Env) && !forcefresh && existed && fileSize > 0 { + return cc.analyzeIncludes(cc.pumpHeadFile) + } + + return nil, ErrorNoPumpHeadFile +} + +func (cc *TaskCC) forceDepend() error { + cc.sourcedependfile = makeTmpFileName(commonUtil.GetHandlerTmpDir(cc.sandbox), "cc_depend", ".d") + cc.sourcedependfile = strings.Replace(cc.sourcedependfile, "\\", "/", -1) + cc.addTmpFile(cc.sourcedependfile) + + cc.forcedepend = true + + return nil +} + +func (cc *TaskCC) inPumpBlack(responseFile string, args []string) (bool, error) { + // obtain black key set by booster + blackkeystr := cc.sandbox.Env.GetEnv(dcEnv.KeyExecutorPumpBlackKeys) + if blackkeystr != "" { + // blog.Infof("cc: got pump black key string: %s", blackkeystr) + blacklist := strings.Split(blackkeystr, dcEnv.CommonBKEnvSepKey) + if len(blacklist) > 0 { + for _, v := range blacklist { + if v != "" && strings.Contains(responseFile, v) { + blog.Infof("cc: found response %s is in pump blacklist", responseFile) + return true, nil + } + + for _, v1 := range args { + if strings.HasSuffix(v1, ".cpp") && strings.Contains(v1, v) { + blog.Infof("cc: found arg %s is in pump blacklist", v1) + return true, nil + } + } + } + } + } + + return false, nil +} + +var ( + // 缓存单个pch文件的依赖列表 + pchDependHeadLock sync.RWMutex + pchDependHead map[string][]*dcFile.Info = make(map[string][]*dcFile.Info, 20) +) + +func setPchDependHead(f string, heads []*dcFile.Info) { + pchDependHeadLock.Lock() + defer pchDependHeadLock.Unlock() + + pchDependHead[f] = heads +} + +func getPchDependHead(f string) []*dcFile.Info { + pchDependHeadLock.RLock() + defer pchDependHeadLock.RUnlock() + + heads, ok := pchDependHead[f] + if ok { + return heads + } + + return nil +} + +func (cc *TaskCC) getPchDepends(fs []*dcFile.Info) ([]*dcFile.Info, error) { + // 取出依赖的所有gch文件 + gchfiles := []string{} + for _, v := range fs { + if strings.HasSuffix(v.Path(), ".gch") { + gchfiles = append(gchfiles, v.Path()) + } + } + + if len(gchfiles) > 0 { + blog.Infof("cc: got pch files:%v", gchfiles) + + gchdependfiles := []string{} + // 再得到这些gch文件的依赖的其它gch文件 + for _, v := range gchfiles { + dfs := dcPump.GetPchDepend(v) + if len(dfs) > 0 { + for _, v1 := range dfs { + newfile := true + for _, v2 := range gchdependfiles { + if v1 == v2 { + newfile = false + break + } + } + + if newfile { + gchdependfiles = append(gchdependfiles, v1) + } + } + } + } + + // 得到最终的 文件信息列表 + if len(gchdependfiles) > 0 { + blog.Infof("cc: got pch depends files:%v", gchdependfiles) + fs, err := commonUtil.GetFileInfo(gchdependfiles, false, false, dcPump.SupportPumpLstatByDir(cc.sandbox.Env)) + if err != nil { + return nil, err + } + + // 获取每个pch的依赖列表 + for _, v := range gchdependfiles { + df, err := cc.getPumpFileByPCHFullPath(v) + if err != nil { + continue + } + + // 尝试从缓存取 + fsv := getPchDependHead(df) + if fsv != nil && len(fsv) > 0 { + fs = append(fs, fsv...) + } else { + fsv, err := cc.analyzeIncludes(df) + if err == nil && len(fsv) > 0 { + fs = append(fs, fsv...) + // 添加到缓存 + setPchDependHead(df, fsv) + } + } + } + + blog.Infof("cc: got pch total %d depends files", len(fs)) + return fs, nil + } + } + + return nil, nil +} + +// first error means real error when try pump, second is notify error +func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, error, error) { + blog.Infof("cc: trypumpwithcache: %v", command) + + // TODO : !! ensureCompilerRaw changed the command slice, it maybe not we need !! + tstart := time.Now().Local() + responseFile, args, showinclude, sourcedependfile, objectfile, pchfile, err := ensureCompilerRaw(command, cc.sandbox.Dir) + if err != nil { + blog.Debugf("cc: pre execute ensure compiler failed %v: %v", args, err) + return nil, err, nil + } else { + blog.Infof("cc: after parse command, got responseFile:%s,sourcedepent:%s,objectfile:%s,pchfile:%s", + responseFile, sourcedependfile, objectfile, pchfile) + } + tend := time.Now().Local() + blog.Debugf("cc: trypumpwithcache time record: %s for ensureCompilerRaw for rsp file:%s", tend.Sub(tstart), responseFile) + tstart = tend + + // 关注 gch 文件的依赖列表 + if strings.HasSuffix(objectfile, ".gch") && sourcedependfile != "" { + cc.needresolvepchdepend = true + cc.sourcedependfile = sourcedependfile + cc.objectpchfile = objectfile + blog.Infof("cc: need resolve dpend for gch file:%s", objectfile) + } + + _, err = scanArgs(args, cc.sandbox) + if err != nil { + blog.Debugf("cc: try pump not support, scan args %v: %v", args, err) + return nil, err, ErrorNotSupportRemote + } + + inblack, _ := cc.inPumpBlack(responseFile, args) + if inblack { + return nil, ErrorInPumpBlack, nil + } + + tend = time.Now().Local() + blog.Debugf("cc: trypumpwithcache time record: %s for scanArgs for rsp file:%s", tend.Sub(tstart), responseFile) + tstart = tend + + if cc.sourcedependfile == "" { + if sourcedependfile != "" { + cc.sourcedependfile = sourcedependfile + } else { + // 主动加上参数得到依赖列表，生成一个临时的 sourcedependfile 文件 + blog.Infof("cc: trypump not found depend file, try append it") + if cc.forceDepend() != nil { + return nil, ErrorNoDependFile, nil + } + } + } + cc.showinclude = showinclude + cc.needcopypumpheadfile = true + + cc.responseFile = responseFile + cc.pumpArgs = args + + includes, err := cc.Includes(responseFile, args, cc.sandbox.Dir, false) + + // 添加pch的依赖列表 + if len(includes) > 0 { + pchdepends, err := cc.getPchDepends(includes) + if err == nil && len(pchdepends) > 0 { + includes = append(includes, pchdepends...) + } + } + + tend = time.Now().Local() + blog.Debugf("cc: trypumpwithcache time record: %s for Includes for rsp file:%s", tend.Sub(tstart), responseFile) + tstart = tend + + if err == nil { + blog.Infof("cc: parse command,got total %d includes files", len(includes)) + + // add pch file as input + if pchfile != "" { + // includes = append(includes, pchfile) + finfo, _ := cc.checkFstat(pchfile, cc.sandbox.Dir) + if finfo != nil { + includes = append(includes, finfo) + } + } + + // add response file as input + if responseFile != "" { + // includes = append(includes, responseFile) + finfo, _ := cc.checkFstat(responseFile, cc.sandbox.Dir) + if finfo != nil { + includes = append(includes, finfo) + } + } + + inputFiles := []dcSDK.FileDesc{} + // priority := dcSDK.MaxFileDescPriority + for _, f := range includes { + // existed, fileSize, modifyTime, fileMode := dcFile.Stat(f).Batch() + existed, fileSize, modifyTime, fileMode := f.Batch() + fpath := f.Path() + if !existed { + err := fmt.Errorf("input file %s not existed", fpath) + blog.Errorf("cc: %v", err) + return nil, err, nil + } + inputFiles = append(inputFiles, dcSDK.FileDesc{ + FilePath: fpath, + Compresstype: protocol.CompressLZ4, + FileSize: fileSize, + Lastmodifytime: modifyTime, + Md5: "", + Filemode: fileMode, + Targetrelativepath: filepath.Dir(fpath), + LinkTarget: f.LinkTarget, + NoDuplicated: true, + // Priority: priority, + }) + // priority++ + // blog.Infof("cc: added include file:%s with modify time %d", fpath, modifyTime) + + blog.Debugf("cc: added include file:%s for object:%s", fpath, objectfile) + } + + results := []string{objectfile} + // add source depend file as result + if sourcedependfile != "" { + results = append(results, sourcedependfile) + } + + // set env which need append to remote + envs := []string{} + for _, v := range cc.sandbox.Env.Source() { + if strings.HasPrefix(v, appendEnvKey) { + envs = append(envs, v) + // set flag we hope append env, not overwrite + flag := fmt.Sprintf("%s=true", dcEnv.GetEnvKey(env.KeyRemoteEnvAppend)) + envs = append(envs, flag) + break + } + } + blog.Infof("cc: env which ready sent to remote:[%v]", envs) + + exeName := command[0] + params := command[1:] + blog.Infof("cc: parse command,server command:[%s %s],dir[%s]", + exeName, strings.Join(params, " "), cc.sandbox.Dir) + return &dcSDK.BKDistCommand{ + Commands: []dcSDK.BKCommand{ + { + WorkDir: cc.sandbox.Dir, + ExePath: "", + ExeName: exeName, + ExeToolChainKey: dcSDK.GetJsonToolChainKey(command[0]), + Params: params, + Inputfiles: inputFiles, + ResultFiles: results, + Env: envs, + }, + }, + CustomSave: true, + }, nil, nil + } + + tend = time.Now().Local() + blog.Debugf("cc: trypumpwithcache time record: %s for return dcSDK.BKCommand for rsp file:%s", tend.Sub(tstart), responseFile) + + return nil, err, nil +} + +func (cc *TaskCC) isPumpActionNumSatisfied() (bool, error) { + minnum := dcPump.PumpMinActionNum(cc.sandbox.Env) + if minnum <= 0 { + return true, nil + } + + curbatchsize := 0 + strsize := cc.sandbox.Env.GetEnv(dcEnv.KeyExecutorTotalActionNum) + if strsize != "" { + size, err := strconv.Atoi(strsize) + if err != nil { + return true, err + } else { + curbatchsize = size + } + } + + blog.Infof("cc: check pump action num with min:%d: current batch num:%d", minnum, curbatchsize) + + return int32(curbatchsize) > minnum, nil +} + +func (cc *TaskCC) workerSupportAbsPath() bool { + v := cc.sandbox.Env.GetEnv(env.KeyWorkerSupportAbsPath) + if v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + } + return true +} + func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { blog.Infof("cc: [%s] start pre execute for: %v", cc.tag, command) cc.originArgs = command + + if !cc.pumpremotefailed && + dcPump.IsPumpCache(cc.sandbox.Env) && + cc.workerSupportAbsPath() { + req, err, notifyerr := cc.trypumpwithcache(command) + if err != nil { + if notifyerr == ErrorNotSupportRemote { + blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err) + return nil, err + } + } else { + // for debug + blog.Debugf("cc: after try pump, req: %+v", *req) + cc.pumpremote = true + return req, err + } + } + compilerEnsuredArgs, err := ensureCompiler(command) if err != nil { blog.Warnf("cc: [%s] pre execute ensure compiler %v: %v", cc.tag, command, err) @@ -217,6 +806,12 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { } } + if cc.forcedepend { + args = append(args, "-MD") + args = append(args, "-MF") + args = append(args, cc.sourcedependfile) + } + if err = cc.preBuild(args); err != nil { blog.Warnf("cc: [%s] pre execute pre-build %v: %v", cc.tag, args, err) return nil, err @@ -268,6 +863,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, error) { ResultFiles: cc.outputFile, }, }, + CustomSave: true, }, nil } @@ -277,11 +873,45 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { return ErrorInvalidParam } + resultfilenum := 0 + // by tomtian 20201224,to ensure existed result file + if len(r.Results[0].ResultFiles) == 0 { + blog.Warnf("cc: not found result file for: %v", cc.originArgs) + goto ERROREND + } + blog.Infof("cc: found %d result files for result[0]", len(r.Results[0].ResultFiles)) + + // resultfilenum := 0 + if len(r.Results[0].ResultFiles) > 0 { + for _, f := range r.Results[0].ResultFiles { + if f.Buffer != nil { + if err := saveResultFile(&f, cc.sandbox.Dir); err != nil { + blog.Errorf("cc: failed to save file [%s]", f.FilePath) + return err + } + resultfilenum++ + } + } + } + + // by tomtian 20201224,to ensure existed result file + if resultfilenum == 0 { + blog.Warnf("cc: not found result file for: %v", cc.originArgs) + goto ERROREND + } + if r.Results[0].RetCode == 0 { blog.Infof("cc: [%s] success done post execute", cc.tag) + // set output to inputFile + r.Results[0].OutputMessage = []byte(filepath.Base(cc.inputFile)) + // if remote succeed with pump,do not need copy head file + if cc.pumpremote { + cc.needcopypumpheadfile = false + } return nil } +ERROREND: // write error message into if cc.saveTemp() && len(r.Results[0].ErrorMessage) > 0 { // make the tmp file for storing the stderr from server compiler. @@ -298,6 +928,11 @@ func (cc *TaskCC) postExecute(r *dcSDK.BKDistResult) error { } } + if cc.pumpremote { + blog.Infof("cc: ready remove pump head file: %s after failed pump remote, generate it next time", cc.pumpHeadFile) + os.Remove(cc.pumpHeadFile) + } + return fmt.Errorf("cc: [%s] failed to remote execute, retcode %d, error message:%s, output message:%s", cc.tag, r.Results[0].RetCode, @@ -340,12 +975,87 @@ func (cc *TaskCC) ensureOwner(fdl []string) { } } +func (cc *TaskCC) resolvePchDepend(workdir string) error { + blog.Infof("cc: ready resolve pch depend file: %s", cc.sourcedependfile) + + sep := "\n" + includes := []string{} + err := cc.resolveDependFile(sep, workdir, &includes) + if err != nil { + return err + } + + if len(includes) > 0 { + if !filepath.IsAbs(cc.objectpchfile) { + cc.objectpchfile, _ = filepath.Abs(filepath.Join(workdir, cc.objectpchfile)) + } + + gchfile := []string{} + for _, v := range includes { + if strings.HasSuffix(v, ".gch") { + newfile := true + for _, v1 := range gchfile { + if v == v1 { + newfile = false + break + } + } + + if newfile { + blog.Infof("cc: found pch depend %s->%s", cc.objectpchfile, v) + gchfile = append(gchfile, v) + } + } + } + + if len(gchfile) > 0 { + dcPump.SetPchDepend(cc.objectpchfile, gchfile) + } + } + + return nil +} + +func (cc *TaskCC) getPumpFileByPCHFullPath(f string) (string, error) { + pumpdir, err := cc.getPumpDir(cc.sandbox.Env) + if err == nil { + args := []string{f} + return getPumpIncludeFile(pumpdir, "pch_depend", ".txt", args, cc.sandbox.Dir) + } + + blog.Warnf("cc: got pch:%s depend file with err:%v", f, err) + return "", err +} + func (cc *TaskCC) finalExecute(args []string, sandbox *dcSyscall.Sandbox) { cc.ensureOwner(getOutputFile(args, sandbox)) - if !cc.saveTemp() { + go func() { + if cc.needcopypumpheadfile { + cc.copyPumpHeadFile(cc.sandbox.Dir) + } + + // 解析pch的依赖关系，并将该gch的依赖列表保存起来 + if cc.needresolvepchdepend { + // 解析并保存pch的依赖关系 + cc.resolvePchDepend(cc.sandbox.Dir) + + // 并将该gch的依赖列表保存起来 + var err error + cc.pumpHeadFile, err = cc.getPumpFileByPCHFullPath(cc.objectpchfile) + if err != nil { + blog.Warnf("cc: failed to get pump head file with pch file:%s err:%v", cc.objectpchfile, err) + } else { + cc.copyPumpHeadFile(cc.sandbox.Dir) + } + } + + if cc.saveTemp() { + return + } + cc.cleanTmpFile() - } + }() } func (cc *TaskCC) saveTemp() bool { @@ -665,7 +1375,7 @@ func (cc *TaskCC) doPreProcess(args []string, inputFile string) (string, []strin // try preprocess with "-E" cc.preProcessArgs = newArgs2 - blog.Debugf("cc: [%s] going to execute pre-process: %s", strings.Join(newArgs2, " ")) + blog.Debugf("cc: [%s] going to execute pre-process: %s", cc.tag, strings.Join(newArgs2, " ")) if _, err = sandbox.ExecCommand(newArgs2[0], newArgs2[1:]...); err != nil { blog.Warnf("cc: [%s] do pre-process %v: %v, %s", cc.tag, newArgs2, err, errBuf.String()) return "", nil, err diff --git a/src/backend/booster/bk_dist/handler/cc/utils.go b/src/backend/booster/bk_dist/handler/cc/utils.go index cd9af717..f49f78af 100644 --- a/src/backend/booster/bk_dist/handler/cc/utils.go +++ b/src/backend/booster/bk_dist/handler/cc/utils.go @@ -10,21 +10,30 @@ package cc import ( + "bufio" "bytes" + "crypto/md5" "fmt" "io/ioutil" "os" "path/filepath" "strings" + "sync" "time" + "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env" + dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" + "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol" + dcPump "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/pump" + dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" - - dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" + dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/codec" commonUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/util" + "github.com/saintfish/chardet" + "golang.org/x/text/encoding/unicode" "github.com/google/shlex" ) @@ -33,6 +42,107 @@ func getEnv(n string) string { return os.Getenv(n) } +func hasSpace(s string) bool { + if s == "" { + return false + } + + for _, v := range s { + if v == ' ' { + return true + } + } + + return false +} + +func checkCharset(rawBytes []byte) (string, error) { + detector := chardet.NewTextDetector() + charset, err := detector.DetectBest(rawBytes) + if err != nil { + return "", err + } + + return charset.Charset, nil +} + +func checkResponseFileCharset(f string) (string, error) { + data, err := ioutil.ReadFile(f) + if err != nil { + return "", err + } + + return checkCharset(data) +} + +func readBom(filename string) (string, error) { + f, err := os.Open(filename) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + //os.Exit(1) + return "", err + } + defer func() { + _ = f.Close() + }() + + dec := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewDecoder() + scn := bufio.NewScanner(dec.Reader(f)) + data := "" + for scn.Scan() { + data = data + scn.Text() + } + if err := scn.Err(); err != nil { + return "", err + } + + return data, nil +} + +func readUtf8(filename string) (string, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return "", err + } + + return string(data), nil +} + +// return compile options and source files +func readResponse(f, dir string) (string, error) { + newf := f + if !dcFile.Stat(newf).Exist() { + // try with dir + tempf, _ := filepath.Abs(filepath.Join(dir, newf)) + if !dcFile.Stat(tempf).Exist() { + return "", fmt.Errorf("%s or %s dose not exist", newf, tempf) + } else { + newf = tempf + } + } + + charset, err := checkResponseFileCharset(newf) + if err != nil { + return "", err + } + + data := "" + if charset == "UTF-16LE" { + data, err = readBom(newf) + } else { + data, err = readUtf8(newf) + } + if err != nil { + return "", err + } + + if data == "" { + return "", fmt.Errorf("%s is empty", newf) + } + + return data, nil +} + // replace which next is not in nextExcludes func replaceWithNextExclude(s string, old byte, new string, nextExcludes []byte) string { if s == "" { @@ -105,6 +215,126 @@ func expandOptions(sandbox *syscall.Sandbox, args []string) ([]string, error) { return newArgs, nil } +// ensure compiler exist in args. +func ensureCompilerRaw(args []string, workdir string) (string, []string, bool, string, string, string, error) { + responseFile := "" + sourcedependfile := "" + objectfile := "" + pchfile := "" + showinclude := false + if len(args) == 0 { + blog.Warnf("cc: ensure compiler got empty arg") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + + if args[0] == "/" || args[0] == "@" || isSourceFile(args[0]) || isObjectFile(args[0]) { + return responseFile, append([]string{defaultCompiler}, args...), showinclude, sourcedependfile, objectfile, pchfile, nil + } + + for _, v := range args { + if strings.HasPrefix(v, "@") { + responseFile = strings.Trim(v[1:], "\"") + + data := "" + if responseFile != "" { + var err error + data, err = readResponse(responseFile, workdir) + if err != nil { + blog.Infof("cc: failed to read response file:%s,err:%v", responseFile, err) + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, err + } + } + // options, sources, err := parseArgument(data) + options, err := shlex.Split(replaceWithNextExclude(string(data), '\\', "\\\\", []byte{'"'})) + if err != nil { + blog.Infof("cc: failed to parse response file:%s,err:%v", responseFile, err) + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, err + } + + args = []string{args[0]} + args = append(args, options...) + + } else if v == "/showIncludes" { + showinclude = true + } + } + + firstinclude := true + for i := range args { + if strings.HasPrefix(args[i], "-MF") { + if len(args[i]) > 3 { + sourcedependfile = args[i][3:] + continue + } + + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -MF") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + sourcedependfile = args[i] + } else if strings.HasPrefix(args[i], "-o") { + // if -o just a prefix, the output file is also in this index, then skip the -o. + if len(args[i]) > 2 { + objectfile = args[i][2:] + blog.Infof("cc: got objectfile file:%s", objectfile) + continue + } + + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -o") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + objectfile = args[i] + blog.Infof("cc: got objectfile file:%s", objectfile) + } else if strings.HasPrefix(args[i], "-include-pch") { + firstinclude = false + if len(args[i]) > 12 { + pchfile = args[i][12:] + continue + } + + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -include-pch") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + pchfile = args[i] + } else if firstinclude && strings.HasPrefix(args[i], "-include") { + firstinclude = false + i++ + if i >= len(args) { + blog.Warnf("cc: scan args: no output file found after -include") + return responseFile, nil, showinclude, sourcedependfile, objectfile, pchfile, ErrorMissingOption + } + pchfile = args[i] + ".gch" + blog.Infof("cc: ready check gch file of %s", pchfile) + } + } + + if responseFile != "" && !filepath.IsAbs(responseFile) { + responseFile, _ = filepath.Abs(filepath.Join(workdir, responseFile)) + } + + if sourcedependfile != "" && !filepath.IsAbs(sourcedependfile) { + sourcedependfile, _ = filepath.Abs(filepath.Join(workdir, sourcedependfile)) + } + + if objectfile != "" && !filepath.IsAbs(objectfile) { + objectfile, _ = filepath.Abs(filepath.Join(workdir, objectfile)) + } + + if pchfile != "" && !filepath.IsAbs(pchfile) { + pchfile, _ = filepath.Abs(filepath.Join(workdir, pchfile)) + if !dcFile.Stat(pchfile).Exist() { + pchfile = "" + } + } + + return responseFile, args, showinclude, sourcedependfile, objectfile, pchfile, nil +} + // ensure compiler exist in args. // change "executor -c foo.c" -> "cc -c foo.c" func ensureCompiler(args []string) ([]string, error) { @@ -216,17 +446,20 @@ var ( // skip options start with flags skipLocalOptionStartWith = map[string]bool{ - "-Wp,": true, - "-Wl,": true, - "-D": true, - "-I": true, - "-U": true, - "-L": true, - "-l": true, - "-MF": true, - "-MT": true, - "-MQ": true, - "-isystem": true, + "-Wp,": true, + "-Wl,": true, + "-D": true, + "-I": true, + "-U": true, + "-L": true, + "-l": true, + "-MF": true, + "-MT": true, + "-MQ": true, + "-isystem": true, + "@": true, // such as @"..\XXX\XXX.rsp" + "--gcc-toolchain": true, + "--sysroot": true, } ) @@ -375,6 +608,9 @@ type ccArgs struct { additionOutputFile []string mfOutputFile []string args []string + includeRspFiles []string // with @ in response file + includePaths []string // with -I + includeFiles []string // with -include } // scanArgs receive the complete compiling args, and the first item should always be a compiler name. @@ -516,6 +752,52 @@ func scanArgs(args []string, sandbox *dcSyscall.Sandbox) (*ccArgs, error) { return nil, ErrorNotSupportDr } + // ++ by tomtian 2021-05-18 + if strings.HasPrefix(arg, "-fsanitize") { + blog.Warnf("cc: scan args: clang option %s need read origin source file; running locally", arg) + return nil, ErrorNotSupportFsanitize + } + // -- + + if strings.HasPrefix(arg, "-I") { + // if -I just a prefix, save the remain of this line. + if len(arg) > 2 { + r.includePaths = append(r.includePaths, strings.Trim(arg[2:], "\"")) + continue + } + + // if file name is in the next index, then take it. + index++ + if index >= len(args) { + blog.Warnf("cc: scan args: no file found after -I") + return nil, ErrorMissingOption + } + r.includePaths = append(r.includePaths, strings.Trim(args[index], "\"")) + continue + } + + if strings.HasPrefix(arg, "-include") { + keylen := 8 + if arg == "-include-pch" { + keylen = 12 + } + + // if -include just a prefix, save the remain of this line. + if len(arg) > keylen { + r.includeFiles = append(r.includeFiles, strings.Trim(arg[keylen:], "\"")) + continue + } + + // if file name is in the next index, then take it. + index++ + if index >= len(args) { + blog.Warnf("cc: scan args: no file found after -include or -include-pch") + return nil, ErrorMissingOption + } + r.includeFiles = append(r.includeFiles, strings.Trim(args[index], "\"")) + continue + } + if strings.HasPrefix(arg, "-o") { // -o should always appear once. if seenOptionO { @@ -541,6 +823,8 @@ func scanArgs(args []string, sandbox *dcSyscall.Sandbox) (*ccArgs, error) { continue } continue + } else if strings.HasPrefix(arg, "@") { + r.includeRspFiles = append(r.includeRspFiles, arg[1:]) } // if this is not start with -, then it maybe a file. @@ -553,6 +837,8 @@ func scanArgs(args []string, sandbox *dcSyscall.Sandbox) (*ccArgs, error) { r.inputFile = arg continue + } else { + blog.Debugf("cc: arg[%s] is not source file", arg) } // if this file is end with .o, it must be the output file. @@ -662,7 +948,8 @@ func outputFromSource(filename, ext string) (string, error) { return "", ErrorInvalidOption } - return strings.TrimSuffix(filename, filepath.Ext(filename)) + ext, nil + // return strings.TrimSuffix(filename, filepath.Ext(filename)) + ext, nil + return strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename)) + ext, nil } // rewrite "cc" to directly call gcc or clang @@ -805,6 +1092,43 @@ func makeTmpFile(tmpDir, prefix, filename string) (string, string, error) { return target, baseDir, nil } +func getPumpIncludeFile(tmpDir, prefix, ext string, args []string, workdir string) (string, error) { + fullarg := strings.Join(args, " ") + md5str := md5.Sum([]byte((fullarg + workdir))) + target := filepath.Join(tmpDir, fmt.Sprintf("%s_%x%s", prefix, md5str, ext)) + + return target, nil +} + +func createFile(target string) error { + for i := 0; i < 3; i++ { + f, err := os.Create(target) + if err != nil { + blog.Errorf("cl: failed to create tmp file \"%s\": %s", target, err) + continue + } + + if err = f.Close(); err != nil { + blog.Errorf("cl: failed to close tmp file \"%s\": %s", target, err) + return err + } + + blog.Infof("cl: success to make tmp file \"%s\"", target) + return nil + } + + return fmt.Errorf("cl: create tmp file failed: %s", target) +} + +// only genegerate file name, do not create really +func makeTmpFileName(tmpDir, prefix, ext string) string { + pid := os.Getpid() + + return filepath.Join(tmpDir, + fmt.Sprintf("%s_%d_%s_%d%s", + prefix, pid, commonUtil.RandomString(8), time.Now().UnixNano(), ext)) +} + // Remove "-o" options from argument list. // // This is used when running the preprocessor, when we just want it to write @@ -903,6 +1227,472 @@ func getFirstIncludeFile(args []string) string { return "" } +func saveResultFile(rf *dcSDK.FileDesc, dir string) error { + fp := rf.FilePath + data := rf.Buffer + blog.Debugf("cc: ready save file [%s]", fp) + if fp == "" { + blog.Warnf("cc: file [%s] path is empty!", fp) + return fmt.Errorf("file path is empty") + } + + if !filepath.IsAbs(fp) { + fp = filepath.Join(dir, fp) + } + + // f, err := os.Create(fp) + // if err != nil { + // if !filepath.IsAbs(fp) && dir != "" { + // newfp, _ := filepath.Abs(filepath.Join(dir, fp)) + // f, err = os.Create(newfp) + // if err != nil { + // blog.Errorf("cc: create file %s or %s error: [%s]", fp, newfp, err.Error()) + // return err + // } + // } else { + // blog.Errorf("cc: create file %s error: [%s]", fp, err.Error()) + // return err + // } + // } + // defer func() { + // _ = f.Close() + // }() + + if rf.CompressedSize > 0 { + switch rf.Compresstype { + case protocol.CompressNone: + + f, err := os.Create(fp) + if err != nil { + if !filepath.IsAbs(fp) && dir != "" { + newfp, _ := filepath.Abs(filepath.Join(dir, fp)) + f, err = os.Create(newfp) + if err != nil { + blog.Errorf("cc: create file %s or %s error: [%s]", fp, newfp, err.Error()) + return err + } + } else { + blog.Errorf("cc: create file %s error: [%s]", fp, err.Error()) + return err + } + } + defer f.Close() + + _, err = f.Write(data) + if err != nil { + blog.Errorf("save file [%s] error: [%s]", fp, err.Error()) + return err + } + break + + case protocol.CompressLZ4: + // decompress with lz4 firstly + dst := make([]byte, rf.FileSize) + if dst == nil { + err := fmt.Errorf("failed to alloc [%d] size buffer", rf.FileSize) + blog.Errorf("%v", err) + return err + } + + // allocTime = time.Now().Local().UnixNano() + outdata, err := dcUtil.Lz4Uncompress(data, dst) + if err != nil { + blog.Errorf("cc: decompress [%s] error: [%s], data len:[%d], buffer len:[%d], filesize:[%d]", + fp, err.Error(), len(data), len(dst), rf.FileSize) + return err + } + // compressTime = time.Now().Local().UnixNano() + // outlen := len(string(outdata)) + outlen := len(outdata) + blog.Debugf("cc: decompressed file %s with lz4, from [%d] to [%d]", fp, rf.CompressedSize, outlen) + if outlen != int(rf.FileSize) { + err := fmt.Errorf("decompressed size %d, expected size %d", outlen, rf.FileSize) + blog.Errorf("cc: decompress error: [%v]", err) + return err + } + + f, err := os.Create(fp) + if err != nil { + if !filepath.IsAbs(fp) && dir != "" { + newfp, _ := filepath.Abs(filepath.Join(dir, fp)) + f, err = os.Create(newfp) + if err != nil { + blog.Errorf("cc: create file %s or %s error: [%s]", fp, newfp, err.Error()) + return err + } + } else { + blog.Errorf("cc: create file %s error: [%s]", fp, err.Error()) + return err + } + } + defer f.Close() + + _, err = f.Write(outdata) + if err != nil { + blog.Errorf("cc: save file [%s] error: [%v]", fp, err) + return err + } + blog.Infof("cc: succeed save file %s size [%d]", fp, outlen) + break + default: + return fmt.Errorf("cc: unknown compress type [%s]", rf.Compresstype) + } + } + + blog.Debugf("cc: succeed to save file [%s]", fp) + return nil +} + +// EscapeArg and MakeCmdLine copied from exec_windows.go + +// EscapeArg rewrites command line argument s as prescribed +// in https://msdn.microsoft.com/en-us/library/ms880421. +// This function returns "" (2 double quotes) if s is empty. +// Alternatively, these transformations are done: +// - every back slash (\) is doubled, but only if immediately +// followed by double quote ("); +// - every double quote (") is escaped by back slash (\); +// - finally, s is wrapped with double quotes (arg -> "arg"), +// but only if there is space or tab inside s. +func EscapeArg(s string) string { + if len(s) == 0 { + return "\"\"" + } + n := len(s) + hasSpace := false + for i := 0; i < len(s); i++ { + switch s[i] { + case '"', '\\': + n++ + case ' ', '\t': + hasSpace = true + } + } + if hasSpace { + n += 2 + } + if n == len(s) { + return s + } + + qs := make([]byte, n) + j := 0 + if hasSpace { + qs[j] = '"' + j++ + } + slashes := 0 + for i := 0; i < len(s); i++ { + switch s[i] { + default: + slashes = 0 + qs[j] = s[i] + case '\\': + slashes++ + qs[j] = s[i] + case '"': + for ; slashes > 0; slashes-- { + qs[j] = '\\' + j++ + } + qs[j] = '\\' + j++ + qs[j] = s[i] + } + j++ + } + if hasSpace { + for ; slashes > 0; slashes-- { + qs[j] = '\\' + j++ + } + qs[j] = '"' + j++ + } + return string(qs[:j]) +} + +// EscapeArg and MakeCmdLine copied from exec_windows.go + +// MakeCmdLine builds a command line out of args by escaping "special" +// characters and joining the arguments with spaces. +func MakeCmdLine(args []string) string { + var s string + for _, v := range args { + if s != "" { + s += " " + } + s += EscapeArg(v) + } + return s +} + +// 根据 clang 命令，获取相应的 resource-dir +type clangResourceDirInfo struct { + clangcommandfullpath string + clangResourceDirpath string +} + +var ( + clangResourceDirlock sync.RWMutex + clangResourceDirs []clangResourceDirInfo +) + +func getResourceDir(cmd string) (string, error) { + var err error + exepfullath := cmd + if !filepath.IsAbs(cmd) { + exepfullath, err = dcUtil.CheckExecutable(cmd) + if err != nil { + return "", err + } + } + + // search from cache + clangResourceDirlock.RLock() + resourcedir := "" + for _, v := range clangResourceDirs { + if exepfullath == v.clangcommandfullpath { + resourcedir = v.clangResourceDirpath + clangResourceDirlock.RUnlock() + return resourcedir, nil + } + } + clangResourceDirlock.RUnlock() + + // try get resource-dir with clang exe path + clangResourceDirlock.Lock() + maxversion := "" + appended := false + defer func() { + // append to cache if not + if !appended { + clangResourceDirs = append(clangResourceDirs, clangResourceDirInfo{ + clangcommandfullpath: exepfullath, + clangResourceDirpath: maxversion, + }) + } + + clangResourceDirlock.Unlock() + }() + + // search from cache again, maybe append by others + for _, v := range clangResourceDirs { + if exepfullath == v.clangcommandfullpath { + resourcedir = v.clangResourceDirpath + appended = true + return resourcedir, nil + } + } + + // real compute resource-dir now + exedir := filepath.Dir(exepfullath) + exeparentdir := filepath.Dir(exedir) + foundclangdir := false + target := filepath.Join(exeparentdir, "lib", "clang") + if dcFile.Stat(target).Exist() { + blog.Infof("cc: found clang dir:%s by exe dir:%s", target, exepfullath) + foundclangdir = true + } else { + target = filepath.Join(exeparentdir, "lib64", "clang") + if dcFile.Stat(target).Exist() { + blog.Infof("cc: found clang dir:%s by exe dir:%s", target, exepfullath) + foundclangdir = true + } + } + + if !foundclangdir { + return resourcedir, fmt.Errorf("not found clang dir") + } + + // get all version dirs, and select the max + files, err := ioutil.ReadDir(target) + if err != nil { + blog.Warnf("failed to get version dirs from dir:%s", target) + return resourcedir, err + } + + versiondirs := []string{} + for _, file := range files { + if file.IsDir() { + nums := strings.Split(file.Name(), ".") + if len(nums) > 1 { + versiondirs = append(versiondirs, filepath.Join(target, file.Name())) + } + } + } + blog.Infof("cc: found all clang version dir:%v", versiondirs) + + if len(versiondirs) == 0 { + return resourcedir, fmt.Errorf("not found any clang's version dir") + } + + maxversion = versiondirs[0] + for _, v := range versiondirs { + if v > maxversion { + maxversion = v + } + } + + blog.Infof("cc: found final resource dir:%s by exe dir:%s", maxversion, exepfullath) + return maxversion, nil +} + +var ( + XcodeIncludeLinkFileslock sync.RWMutex + XcodeIncludeReal2link = make(map[string]string, 0) + XcodeIncludeLink2real = make(map[string]string, 0) + XcodeIncludeLinkResolved = false +) + +func getIncludeLinks(env *env.Sandbox, uniqlines []string) ([]string, error) { + if !dcPump.SupportPumpSearchLink(env) { + return nil, nil + } + + if !XcodeIncludeLinkResolved { + XcodeIncludeLinkFileslock.Lock() + + if !XcodeIncludeLinkResolved { + XcodeIncludeLinkResolved = true + + var err error + resultfile := dcPump.LinkResultFile(env) + XcodeIncludeLink2real, XcodeIncludeReal2link, err = dcPump.ResolveLinkData(resultfile) + if err != nil { + blog.Infof("cc: resolve link file %s with error:%v", resultfile, err) + } + } + + XcodeIncludeLinkFileslock.Unlock() + } + + if XcodeIncludeLink2real != nil { + temparr := make([]string, 0, 10) + for _, l := range uniqlines { + if v, ok := XcodeIncludeLink2real[l]; ok { + temparr = append(temparr, v) + } + if v, ok := XcodeIncludeReal2link[l]; ok { + temparr = append(temparr, v) + } + } + return temparr, nil + } + + return nil, nil + +} + +// scanRspFiles 类似scanArgs，递归解析包含的rsp文件，得到依赖列表，包括路径/文件/新的rsp列表 +func scanRspFilesRecursively( + newrspfile string, + workdir string, + resultIncludePaths *[]string, + resultIncludeFiles *[]string, + checkedRspFiles *[]string) { + blog.Infof("cc: ready resolve recursively rsp file: %s", newrspfile) + + for _, f := range *checkedRspFiles { + if f == newrspfile { + blog.Errorf("cc: found dead loop include response file %s", newrspfile) + return + } + } + + *checkedRspFiles = append(*checkedRspFiles, newrspfile) + + if !filepath.IsAbs(newrspfile) { + newrspfile, _ = filepath.Abs(filepath.Join(workdir, newrspfile)) + } + + blog.Infof("cc: ready resolve recursively rsp file with full path: %s", newrspfile) + + data := "" + var err error + data, err = readResponse(newrspfile, workdir) + if err != nil { + blog.Infof("cc: failed to read response file:%s,err:%v", newrspfile, err) + return + } + + // options, sources, err := parseArgument(data) + args, err := shlex.Split(replaceWithNextExclude(string(data), '\\', "\\\\", []byte{'"'})) + if err != nil { + blog.Infof("cc: failed to parse response file:%s,err:%v", newrspfile, err) + return + } + + // for debug + blog.Infof("cc: response file:%s,args:%+v", newrspfile, args) + + // 只关心包含的依赖，其它选项忽略 + for index := 0; index < len(args); index++ { + arg := args[index] + if strings.HasPrefix(arg, "-") { + if strings.HasPrefix(arg, "-I") { + // if -I just a prefix, save the remain of this line. + if len(arg) > 2 { + *resultIncludePaths = append(*resultIncludePaths, strings.Trim(arg[2:], "\"")) + // for debug + blog.Debugf("cc: response file:%s,got include path:%s", newrspfile, strings.Trim(arg[2:], "\"")) + continue + } + + // if file name is in the next index, then take it. + index++ + if index >= len(args) { + blog.Warnf("cc: scan args: no file found after -I") + return + } + *resultIncludePaths = append(*resultIncludePaths, strings.Trim(args[index], "\"")) + // for debug + blog.Debugf("cc: response file:%s,got include path:%s", newrspfile, strings.Trim(args[index], "\"")) + continue + } + + if strings.HasPrefix(arg, "-include") { + keylen := 8 + if arg == "-include-pch" { + keylen = 12 + } + + // if -include just a prefix, save the remain of this line. + if len(arg) > keylen { + *resultIncludeFiles = append(*resultIncludeFiles, strings.Trim(arg[keylen:], "\"")) + // for debug + blog.Debugf("cc: response file:%s,got include file:%s", newrspfile, strings.Trim(arg[keylen:], "\"")) + continue + } + + // if file name is in the next index, then take it. + index++ + if index >= len(args) { + blog.Warnf("cc: scan args: no file found after -include or -include-pch") + return + } + *resultIncludeFiles = append(*resultIncludeFiles, strings.Trim(args[index], "\"")) + // for debug + blog.Debugf("cc: response file:%s,got include file:%s", newrspfile, strings.Trim(args[index], "\"")) + continue + } + continue + } else if strings.HasPrefix(arg, "@") { + // 递归调用 + scanRspFilesRecursively( + arg[1:], + workdir, + resultIncludePaths, + resultIncludeFiles, + checkedRspFiles) + } + } + + // for debug + blog.Debugf("cc: response file:%s,resultIncludePaths:%+v,resultIncludeFiles:%+v", + newrspfile, *resultIncludePaths, *resultIncludeFiles) +} + func getOutputFile(args []string, sandbox *dcSyscall.Sandbox) []string { r := make([]string, 0, 10) seenOptionO := false diff --git a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler_long_tcp.go b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler_long_tcp.go index 82e136e9..02afadeb 100644 --- a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler_long_tcp.go +++ b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_handler_long_tcp.go @@ -81,7 +81,7 @@ func (r *CommonRemoteHandler) executeTaskLongTCP( return nil, err } - blog.Debugf("protocol: execute dist task commands: %v", req.Commands) + blog.Infof("protocol: execute dist task commands: %v", req.Commands) r.recordStats.RemoteWorkTimeoutSec = r.ioTimeout // var err error diff --git a/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go b/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go index 315840d9..71c1859e 100644 --- a/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go +++ b/src/backend/booster/bk_dist/worker/pkg/protocol/pb_protocol.go @@ -439,9 +439,21 @@ func saveFile( blog.Warnf("create dir %s before create input symlink %s failed: %v", inputDir, inputfile, err) return "", err } - if err = os.Symlink(linkTarget, inputfile); err != nil && !os.IsExist(err) { - blog.Errorf("create input symlink %s -> %s error: [%s]", inputfile, linkTarget, err.Error()) - return "", err + if err = os.Symlink(linkTarget, inputfile); err != nil { + if os.IsExist(err) { + err = os.Remove(inputfile) + if err != nil { + blog.Errorf("remove old symlink %s with error:%v", inputfile, err) + return "", err + } + if err = os.Symlink(linkTarget, inputfile); err != nil { + blog.Errorf("create input symlink %s -> %s error: [%s]", inputfile, linkTarget, err.Error()) + return "", err + } + } else { + blog.Errorf("create input symlink %s -> %s error: [%s]", inputfile, linkTarget, err.Error()) + return "", err + } } // if the link is a dir, then we should ensure the target exist.