diff --git a/src/backend/booster/bk_dist/booster/command/command.go b/src/backend/booster/bk_dist/booster/command/command.go index 33b101e57..67972c82e 100644 --- a/src/backend/booster/bk_dist/booster/command/command.go +++ b/src/backend/booster/bk_dist/booster/command/command.go @@ -63,6 +63,7 @@ const ( FlagOutputEnvSourceFile = "output_env_source_file" FlagCommitSuicide = "commit_suicide" FlagToolChainJSONFile = "tool_chain_json_file" + FlagSearchToolchain = "search_toolchain" FlagBatchMode = "batch_mode" FlagDirectives = "directives" FlagGlobalSlots = "global_slots" @@ -279,6 +280,10 @@ var ( Name: "tool_chain_json_file", Usage: "json file to describe tool chain", }, + commandCli.BoolFlag{ + Name: "search_toolchain", + Usage: "automatically search for toolchain based on command", + }, commandCli.BoolFlag{ Name: "batch_mode, bm", Usage: "batch mode for booster, multi booster in the same projectID will use one same workID", diff --git a/src/backend/booster/bk_dist/booster/command/process.go b/src/backend/booster/bk_dist/booster/command/process.go index f0594aa9d..c4375a30b 100644 --- a/src/backend/booster/bk_dist/booster/command/process.go +++ b/src/backend/booster/bk_dist/booster/command/process.go @@ -315,6 +315,7 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) { WriteMemroy: c.Bool(FlagWriteMemroMemroy), IdleKeepSecs: c.Int(FlagIdleKeepSecs), CleanTmpFilesDayAgo: cleanTmpFilesDayAgo, + SearchToolchain: c.Bool(FlagSearchToolchain), }, Transport: dcType.BoosterTransport{ diff --git a/src/backend/booster/bk_dist/booster/pkg/booster.go b/src/backend/booster/bk_dist/booster/pkg/booster.go index badd260a0..ef2d1163f 100644 --- a/src/backend/booster/bk_dist/booster/pkg/booster.go +++ b/src/backend/booster/bk_dist/booster/pkg/booster.go @@ -360,6 +360,10 @@ func (b *Booster) getWorkersEnv() map[string]string { requiredEnv[env.KeyExecutorIdleKeepSecs] = strconv.Itoa(b.config.Works.IdleKeepSecs) } + if b.config.Works.SearchToolchain { + requiredEnv[env.KeyExecutorSearchToolchain] = envValueTrue + } + resultEnv := make(map[string]string, 10) for k, v := range requiredEnv { resultEnv[env.GetEnvKey(k)] = v diff --git a/src/backend/booster/bk_dist/common/env/env.go b/src/backend/booster/bk_dist/common/env/env.go index 9b70e08a3..83313817b 100644 --- a/src/backend/booster/bk_dist/common/env/env.go +++ b/src/backend/booster/bk_dist/common/env/env.go @@ -64,6 +64,7 @@ const ( KeyExecutorTotalActionNum = "TOTAL_ACTION_NUM" KeyExecutorUseWebSocket = "USE_WEBSOCKET" KeyExecutorNewShader = "NEW_SHADER" + KeyExecutorSearchToolchain = "SEARCH_TOOLCHAIN" KeyUserDefinedLogLevel = "USER_DEFINED_LOG_LEVEL" KeyUserDefinedExecutorLogLevel = "USER_DEFINED_EXECUTOR_LOG_LEVEL" diff --git a/src/backend/booster/bk_dist/common/syscall/syscall_darwin.go b/src/backend/booster/bk_dist/common/syscall/syscall_darwin.go index 49c9f01c7..114ea37ef 100644 --- a/src/backend/booster/bk_dist/common/syscall/syscall_darwin.go +++ b/src/backend/booster/bk_dist/common/syscall/syscall_darwin.go @@ -331,3 +331,7 @@ func RedirectStderror(f string) error { return nil } + +func NeedSearchToolchain(input *env.Sandbox) bool { + return false +} diff --git a/src/backend/booster/bk_dist/common/syscall/syscall_unix.go b/src/backend/booster/bk_dist/common/syscall/syscall_unix.go index 638ddcd89..513444c69 100644 --- a/src/backend/booster/bk_dist/common/syscall/syscall_unix.go +++ b/src/backend/booster/bk_dist/common/syscall/syscall_unix.go @@ -332,3 +332,11 @@ func RedirectStderror(f string) error { return nil } + +func NeedSearchToolchain(input *env.Sandbox) bool { + if input != nil { + return input.GetEnv(env.KeyExecutorSearchToolchain) != "" + } + + return env.GetEnv(env.KeyExecutorSearchToolchain) != "" +} diff --git a/src/backend/booster/bk_dist/common/syscall/syscall_windows.go b/src/backend/booster/bk_dist/common/syscall/syscall_windows.go index 46e3d2cbb..3ae87c04c 100644 --- a/src/backend/booster/bk_dist/common/syscall/syscall_windows.go +++ b/src/backend/booster/bk_dist/common/syscall/syscall_windows.go @@ -516,3 +516,7 @@ func RedirectStderror(f string) error { return nil } + +func NeedSearchToolchain(input *env.Sandbox) bool { + return false +} diff --git a/src/backend/booster/bk_dist/common/types/booster.go b/src/backend/booster/bk_dist/common/types/booster.go index 9a6d787e7..d69e94f74 100644 --- a/src/backend/booster/bk_dist/common/types/booster.go +++ b/src/backend/booster/bk_dist/common/types/booster.go @@ -132,6 +132,8 @@ type BoosterWorks struct { EnableLink bool EnableLib bool + + SearchToolchain bool } // BoosterTransport describe the transport data to controller diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go index 90adef137..0acb0bce1 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go @@ -42,13 +42,14 @@ func NewMgr(pCtx context.Context, work *types.Work) types.BasicMgr { ctx, _ := context.WithCancel(pCtx) return &Mgr{ - ctx: ctx, - work: work, - settings: &types.WorkSettings{}, - info: types.NewInitWorkInfo(work.ID()), - waitWorkReadyTick: 100 * time.Millisecond, - analysisStatus: types.NewWorkAnalysisStatus(), - toolchainMap: make(map[string]toolchainCache, 1), + ctx: ctx, + work: work, + settings: &types.WorkSettings{}, + info: types.NewInitWorkInfo(work.ID()), + waitWorkReadyTick: 100 * time.Millisecond, + analysisStatus: types.NewWorkAnalysisStatus(), + toolchainMap: make(map[string]toolchainCache, 1), + searchToolChainCache: make(map[string]bool, 10), } } @@ -70,9 +71,13 @@ type Mgr struct { toolchainLock sync.RWMutex toolchainMap map[string]toolchainCache + searchToolchainLock sync.RWMutex + aliveTask int64 registeredCounter int32 + + searchToolChainCache map[string]bool } // Alive return the current alive task number @@ -516,3 +521,24 @@ func (m *Mgr) GetToolChainTimestamp(key string) (int64, error) { return v.toolchain.Timestamp, nil } + +// SearchToolChain search toolchain files by cmd, ensure only execute once +func (m *Mgr) SearchToolChain(cmd string) error { + m.searchToolchainLock.Lock() + defer m.searchToolchainLock.Unlock() + + if _, ok := m.searchToolChainCache[cmd]; ok { + blog.Infof("basic: toolchain with key:%s search before, do nothing", cmd) + return nil + } + + m.searchToolChainCache[cmd] = true + + // TODO : search toolchain with cmd now + toolchain, err := searchToolChain(cmd) + if err == nil && toolchain != nil { + m.SetToolChain(toolchain) + } + + return nil +} diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go b/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go index ae669c411..e7640bd34 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go @@ -11,10 +11,14 @@ package basic import ( "fmt" + "os" "path/filepath" "strings" + "time" + dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" + dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/controller/pkg/types" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" @@ -209,3 +213,309 @@ func getToolchainID() string { uniqRemoteToolchainID = fmt.Sprintf("tc_%s", dcUtil.UniqID()) return uniqRemoteToolchainID } + +// ++++++++++++++++++++to search toolchain++++++++++++++++++++ + +func searchToolChain(cmd string) (*types.ToolChain, error) { + blog.Infof("basic: real start search toolchian for cmd:%s", cmd) + defer blog.Infof("basic: end search toolchian for cmd:%s", cmd) + + if strings.HasSuffix(cmd, "clang") || strings.HasSuffix(cmd, "clang++") { + return searchClang(cmd) + } + + if strings.HasSuffix(cmd, "gcc") || strings.HasSuffix(cmd, "g++") { + return searchGcc(cmd) + } + + return nil, nil +} + +func searchCC(exe string) []string { + // search cc1 + cmd := fmt.Sprintf("%s -print-prog-name=cc1", exe) + blog.Infof("basic: ready run cmd:[%s] for exe:%s", cmd, exe) + sandbox := dcSyscall.Sandbox{} + _, out, _, err := sandbox.ExecScriptsWithMessage(cmd) + if err != nil { + blog.Warnf("basic: search as with out:%s,error:%+v", out, err) + return nil + } + + blog.Infof("basic: got output:[%s] for cmd:[%s]", out, cmd) + + cc1 := strings.TrimSpace(string(out)) + i := dcFile.Lstat(cc1) + if !i.Exist() { + err := fmt.Errorf("file %s not existed", cc1) + blog.Errorf("basic: %v", err) + return nil + } + + // cc1plus + cc1plus := cc1 + "plus" + i = dcFile.Lstat(cc1plus) + if !i.Exist() { + err := fmt.Errorf("file %s not existed", cc1plus) + blog.Errorf("basic: %v", err) + return nil + } + + return []string{cc1, cc1plus} +} + +func searchAS(exe string) []string { + cmd := "which as" + blog.Infof("basic: ready run cmd:[%s] for exe:%s", cmd, exe) + sandbox := dcSyscall.Sandbox{} + _, out, _, err := sandbox.ExecScriptsWithMessage(cmd) + if err != nil { + blog.Warnf("basic: search as with out:%s,error:%+v", out, err) + return nil + } + + blog.Infof("basic: got output:[%s] for cmd:[%s]", out, cmd) + f := strings.TrimSpace(string(out)) + i := dcFile.Lstat(f) + if !i.Exist() { + err := fmt.Errorf("file %s not existed", f) + blog.Errorf("basic: %v", err) + return nil + } + + return []string{f} +} + +var soWhiteList = []string{ + "libstdc++", + "libmpc", + "libmpfr", + "libgmp", + "libopcodes", + "libbfd", +} + +var specialSOFiles = []string{ + "/lib64/libstdc++.so.6", +} + +func getLddFiles(exe string) []string { + cmd := fmt.Sprintf("ldd %s", exe) + blog.Infof("basic: ready run cmd:[%s] for exe:%s", cmd, exe) + sandbox := dcSyscall.Sandbox{} + _, out, _, err := sandbox.ExecScriptsWithMessage(cmd) + if err != nil { + blog.Warnf("basic: search as with out:%s,error:%+v", out, err) + return nil + } + + blog.Infof("basic: got output:[%s] for cmd:[%s]", out, cmd) + fields := strings.Fields(string(out)) + files := []string{} + for _, f := range fields { + // 这个好像会导致异常，先屏蔽 + if strings.HasSuffix(f, "libc.so.6") { + continue + } + if filepath.IsAbs(f) && strings.Contains(f, ".so") { + inWhiteList := false + for _, w := range soWhiteList { + if strings.Contains(f, w) { + inWhiteList = true + break + } + } + + if inWhiteList && dcFile.Lstat(f).Exist() { + files = append(files, f) + } + } + } + + blog.Infof("basic: got ldd files:%v for exe:%s", files, exe) + return files +} + +func searchGcc(cmd string) (*types.ToolChain, error) { + blog.Infof("basic: search gcc toolchain with exe:%s", cmd) + + i := dcFile.Lstat(cmd) + if !i.Exist() { + err := fmt.Errorf("cmd %s not existed", cmd) + blog.Errorf("basic: %v", err) + return nil, err + } + + t := &types.ToolChain{ + ToolKey: cmd, + ToolName: filepath.Base(cmd), + ToolLocalFullPath: cmd, + ToolRemoteRelativePath: filepath.Dir(cmd), + Files: make([]dcSDK.ToolFile, 0), + Timestamp: time.Now().Local().UnixNano(), + } + + exefiles := []string{cmd} + + // search cc1 / cc1plus + fs := searchCC(cmd) + for _, i := range fs { + t.Files = append(t.Files, dcSDK.ToolFile{ + LocalFullPath: i, + RemoteRelativePath: filepath.Dir(i), + }) + exefiles = append(exefiles, i) + } + + // search as + fs = searchAS(cmd) + for _, i := range fs { + t.Files = append(t.Files, dcSDK.ToolFile{ + LocalFullPath: i, + RemoteRelativePath: filepath.Dir(i), + }) + exefiles = append(exefiles, i) + } + + // ldd files for executable files + for _, i := range exefiles { + fs = getLddFiles(i) + for _, i := range fs { + t.Files = append(t.Files, dcSDK.ToolFile{ + LocalFullPath: i, + RemoteRelativePath: filepath.Dir(i), + }) + } + } + + // add special so + for _, f := range specialSOFiles { + if dcFile.Lstat(f).Exist() { + t.Files = append(t.Files, dcSDK.ToolFile{ + LocalFullPath: f, + RemoteRelativePath: filepath.Dir(f), + }) + } + } + + blog.Infof("basic: got gcc/g++ toolchian:%+v", *t) + return t, nil +} + +// search clang crtbegin.o +func searchCrtbegin(exe string) []string { + cmd := fmt.Sprintf("%s -v", exe) + blog.Infof("basic: ready run cmd:[%s]", cmd) + sandbox := dcSyscall.Sandbox{} + _, out, errmsg, err := sandbox.ExecScriptsWithMessage(cmd) + if err != nil { + blog.Warnf("basic: search clang crtbegin with out:%s,error:%+v", out, err) + return nil + } + blog.Infof("basic: got output:[%s] errmsg:[%s] for cmd:[%s]", out, errmsg, cmd) + + // resolve output message + gccpath := "" + key := "Selected GCC installation:" + lines := strings.Split(string(errmsg), "\n") + for _, l := range lines { + blog.Infof("basic: check line:[%s]", l) + if strings.HasPrefix(l, key) { + fields := strings.Split(l, ":") + blog.Infof("basic: got fields:%+v,len(fields):%d", fields, len(fields)) + if len(fields) == 2 { + gccpath = strings.TrimSpace(fields[1]) + i := dcFile.Lstat(gccpath) + if !i.Exist() { + err := fmt.Errorf("path %s not existed", gccpath) + blog.Errorf("basic: %v", err) + return nil + } else { + blog.Infof("basic: gcc path:[%s] existed", gccpath) + } + } + break + } + } + + fs := make([]string, 0, 2) + if gccpath != "" { + err := filepath.Walk(gccpath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasPrefix(info.Name(), "crt") && strings.HasSuffix(info.Name(), ".o") { + fs = append(fs, path) + } + return nil + }) + + if err != nil { + blog.Warnf("basic: walk dir:%s with error:%v", gccpath, err) + } + } + + if len(fs) > 0 { + blog.Infof("basic: got crtbegin files:%v", fs) + return fs + } + + return nil +} + +func searchClang(cmd string) (*types.ToolChain, error) { + blog.Infof("basic: search clang toolchain with exe:%s", cmd) + + i := dcFile.Lstat(cmd) + if !i.Exist() { + err := fmt.Errorf("cmd %s not existed", cmd) + blog.Errorf("basic: %v", err) + return nil, err + } + + t := &types.ToolChain{ + ToolKey: cmd, + ToolName: filepath.Base(cmd), + ToolLocalFullPath: cmd, + ToolRemoteRelativePath: filepath.Dir(cmd), + Files: make([]dcSDK.ToolFile, 0), + Timestamp: time.Now().Local().UnixNano(), + } + + // search clang crtbegin.o + fs := searchCrtbegin(cmd) + for _, i := range fs { + t.Files = append(t.Files, dcSDK.ToolFile{ + LocalFullPath: i, + RemoteRelativePath: filepath.Dir(i), + }) + } + + exefiles := []string{cmd} + + // ldd files for executable files + for _, i := range exefiles { + fs = getLddFiles(i) + for _, i := range fs { + t.Files = append(t.Files, dcSDK.ToolFile{ + LocalFullPath: i, + RemoteRelativePath: filepath.Dir(i), + }) + } + } + + // add special so + for _, f := range specialSOFiles { + if dcFile.Lstat(f).Exist() { + t.Files = append(t.Files, dcSDK.ToolFile{ + LocalFullPath: f, + RemoteRelativePath: filepath.Dir(f), + }) + } + } + + blog.Infof("basic: got clang/clang++ toolchian:%+v", *t) + return t, nil +} + +// --------------------to search toolchain-------------------- diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go index 29602a5cb..84e807fd7 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go @@ -1649,8 +1649,23 @@ func (m *Mgr) getToolChainFromExecuteRequest(req *types.RemoteTaskExecuteRequest fd = append(fd, additionfd) } if c.ExeToolChainKey != "" { - if toolfd := m.getToolFileInfoByKey(c.ExeToolChainKey); toolfd != nil { + toolfd := m.getToolFileInfoByKey(c.ExeToolChainKey) + if toolfd != nil { fd = append(fd, toolfd) + } else { + // TODO : 如果环境变量中指定了需要自动探测工具链，则需要自动探测 + if dcSyscall.NeedSearchToolchain(req.Sandbox.Env) { + blog.Infof("remote: start search toolchain with key:%s now", c.ExeToolChainKey) + err := m.work.Basic().SearchToolChain(c.ExeToolChainKey) + blog.Infof("remote: end search toolchain with key:%s error:%v", c.ExeToolChainKey, err) + + if err == nil { + toolfd = m.getToolFileInfoByKey(c.ExeToolChainKey) + if toolfd != nil { + fd = append(fd, toolfd) + } + } + } } } } diff --git a/src/backend/booster/bk_dist/controller/pkg/types/interface.go b/src/backend/booster/bk_dist/controller/pkg/types/interface.go index 887e5dd15..518824816 100644 --- a/src/backend/booster/bk_dist/controller/pkg/types/interface.go +++ b/src/backend/booster/bk_dist/controller/pkg/types/interface.go @@ -265,4 +265,7 @@ type BasicMgr interface { // minus registered count for batch mode DecRegistered() + + // search toolchain files by cmd, ensure only execute once + SearchToolChain(cmd string) error } diff --git a/src/backend/booster/bk_dist/executor/pkg/dist_executor.go b/src/backend/booster/bk_dist/executor/pkg/dist_executor.go index cc721085d..7b409950f 100644 --- a/src/backend/booster/bk_dist/executor/pkg/dist_executor.go +++ b/src/backend/booster/bk_dist/executor/pkg/dist_executor.go @@ -175,6 +175,42 @@ func runDirect() (int, string, error) { return exitCode, "", err } +func getAbsPath(cmd string) string { + // absolute path + if filepath.IsAbs(cmd) { + return cmd + } + + // relative path + if strings.Contains(cmd, "/") { + dir, _ := os.Getwd() + if realDir, err := filepath.EvalSymlinks(dir); err == nil { + dir = realDir + } + + newcmd := filepath.Join(dir, cmd) + if !filepath.IsAbs(newcmd) { + var err error + newcmd, err = filepath.Abs(newcmd) + if err == nil { + return newcmd + } else { + blog.Errorf("executor: get abs path for %s with error:%v", newcmd, err) + return cmd + } + } + } + + // no path, only file name + newcmd, err := exec.LookPath(cmd) + if err == nil { + return newcmd + } else { + blog.Errorf("executor: LookPath for %s with error:%v", cmd, err) + return cmd + } +} + func (d *DistExecutor) runWork() (int, string, error) { d.initStats() @@ -183,6 +219,11 @@ func (d *DistExecutor) runWork() (int, string, error) { return 0, "", fmt.Errorf("not enough args to execute") } + // TODO : 如果支持了自动获取工具链，这儿将命令转为绝对路径 + if dcSyscall.NeedSearchToolchain(nil) { + os.Args[1] = getAbsPath(os.Args[1]) + } + retcode, retmsg, r, err := d.work.Job(d.stats).ExecuteLocalTask(os.Args[1:], "") if err != nil || retcode != 0 { if r != nil { diff --git a/src/backend/booster/bk_dist/handler/cc/booster.go b/src/backend/booster/bk_dist/handler/cc/booster.go index f52a8990c..6e397de8a 100644 --- a/src/backend/booster/bk_dist/handler/cc/booster.go +++ b/src/backend/booster/bk_dist/handler/cc/booster.go @@ -90,6 +90,13 @@ func (cc *TaskCC) appendPump(config dcType.BoosterConfig) error { return nil } +func (cc *TaskCC) appendSearchToolchain(config dcType.BoosterConfig) error { + if config.Works.SearchToolchain { + bazelActionConstOptions = append(bazelActionConstOptions, env.GetEnvKey(env.KeyExecutorSearchToolchain)) + } + return nil +} + // ProjectExtraData describe the extra data store in project // ccache_enable and ccache_enabled are both to control ccache usage, if one of them is true, then ccache enabled. type ProjectExtraData struct { @@ -167,6 +174,7 @@ func (cc *TaskCC) RenderArgs(config dcType.BoosterConfig, originArgs string) str appendPreload() cc.appendCcache(config) cc.appendPump(config) + cc.appendSearchToolchain(config) if config.Works.BazelPlus || config.Works.Bazel4Plus || config.Works.BazelNoLauncher { additions := make([]string, 0, 10) diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go index e9b6ec103..56a06bd2d 100644 --- a/src/backend/booster/bk_dist/handler/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/cc/handler.go @@ -307,6 +307,12 @@ func (cc *TaskCC) resolveDependFile(sep, workdir string, includes *[]string) err } *includes = append(*includes, commonUtil.FormatFilePath(targetf)) + + // 如果是链接，则将相关指向的文件都包含进来 + fs := commonUtil.GetAllLinkFiles(targetf, workdir) + if len(fs) > 0 { + *includes = append(*includes, fs...) + } } } } diff --git a/src/backend/booster/bk_dist/handler/common/util.go b/src/backend/booster/bk_dist/handler/common/util.go index f09baa94e..46566b9df 100644 --- a/src/backend/booster/bk_dist/handler/common/util.go +++ b/src/backend/booster/bk_dist/handler/common/util.go @@ -114,52 +114,65 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo // query tempis := make(map[string]*dcFile.Info, len(notfound)) - for _, f := range notfound { - var i *dcFile.Info - if statbysearchdir { - i = dcFile.GetFileInfoByEnumDir(f) - } else { - i = dcFile.Lstat(f) - } - tempis[f] = i - - if !i.Exist() { - if mustexisted { - // TODO : return fail if not existed - // continue - blog.Warnf("common util: depend file:%s not existed ", f) - return nil, fmt.Errorf("%s not existed", f) + for _, notf := range notfound { + tempf := notf + for { + var i *dcFile.Info + if statbysearchdir { + i = dcFile.GetFileInfoByEnumDir(tempf) } else { - continue + i = dcFile.Lstat(tempf) + } + tempis[tempf] = i + + if !i.Exist() { + if mustexisted { + // TODO : return fail if not existed + // continue + blog.Warnf("common util: depend file:%s not existed ", tempf) + return nil, fmt.Errorf("%s not existed", tempf) + } else { + // continue + break + } } - } - if i.Basic().Mode()&os.ModeSymlink != 0 { - originFile, err := os.Readlink(f) - if err == nil { - if !filepath.IsAbs(originFile) { - originFile, err = filepath.Abs(filepath.Join(filepath.Dir(f), originFile)) - if err == nil { - i.LinkTarget = originFile - blog.Infof("common util: symlink %s to %s", f, originFile) + loopagain := false + if i.Basic().Mode()&os.ModeSymlink != 0 { + originFile, err := os.Readlink(tempf) + if err == nil { + if !filepath.IsAbs(originFile) { + originFile, err = filepath.Abs(filepath.Join(filepath.Dir(tempf), originFile)) + if err == nil { + i.LinkTarget = originFile + blog.Infof("common util: symlink %s to %s", tempf, originFile) + } else { + blog.Infof("common util: symlink %s origin %s, got abs path error:%s", + tempf, originFile, err) + } } else { - blog.Infof("common util: symlink %s origin %s, got abs path error:%s", - f, originFile, err) + i.LinkTarget = originFile + blog.Infof("common util: symlink %s to %s", tempf, originFile) } + + // 如果是链接，并且指向了其它文件，则需要将指向的文件也包含进来 + loopagain = true + tempf = originFile } else { - i.LinkTarget = originFile - blog.Infof("common util: symlink %s to %s", f, originFile) + blog.Infof("common util: symlink %s Readlink error:%s", tempf, err) } - } else { - blog.Infof("common util: symlink %s Readlink error:%s", f, err) } - } - if notdir && i.Basic().IsDir() { - continue - } + if notdir && i.Basic().IsDir() { + continue + } - is = append(is, i) + is = append(is, i) + + if !loopagain { + break + } + } } // write @@ -341,3 +354,30 @@ func FormatFilePath(f string) string { return f } + +func GetAllLinkFiles(f, workdir string) []string { + fs := []string{} + tempf := f + for { + loopagain := false + i := dcFile.Lstat(tempf) + if i.Basic().Mode()&os.ModeSymlink != 0 { + originFile, err := os.Readlink(tempf) + if err == nil { + if !filepath.IsAbs(originFile) { + originFile, _ = filepath.Abs(filepath.Join(workdir, originFile)) + } + fs = append(fs, FormatFilePath(originFile)) + + loopagain = true + tempf = originFile + } + } + + if !loopagain { + break + } + } + + return fs +} diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go index 4dc2d32ef..a11e111bb 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go @@ -355,6 +355,12 @@ func (cc *TaskCC) resolveDependFile(sep, workdir string, includes *[]string) err } *includes = append(*includes, commonUtil.FormatFilePath(targetf)) + + // 如果是链接，则将相关指向的文件都包含进来 + fs := commonUtil.GetAllLinkFiles(targetf, workdir) + if len(fs) > 0 { + *includes = append(*includes, fs...) + } } } } diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go index 6f87236c0..9d1041069 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go @@ -484,6 +484,12 @@ func (cl *TaskCL) copyPumpHeadFile(workdir string) error { l, _ = filepath.Abs(filepath.Join(workdir, l)) } includes = append(includes, commonUtil.FormatFilePath(l)) + + // 如果是链接，则将相关指向的文件都包含进来 + fs := commonUtil.GetAllLinkFiles(l, workdir) + if len(fs) > 0 { + includes = append(includes, fs...) + } } } else { blog.Warnf("cl: failed to resolve depend file: %s with err:%s", cl.sourcedependfile, err) @@ -497,6 +503,12 @@ func (cl *TaskCL) copyPumpHeadFile(workdir string) error { l, _ = filepath.Abs(filepath.Join(workdir, l)) } includes = append(includes, commonUtil.FormatFilePath(l)) + + // 如果是链接，则将相关指向的文件都包含进来 + fs := commonUtil.GetAllLinkFiles(l, workdir) + if len(fs) > 0 { + includes = append(includes, fs...) + } } }