diff --git a/src/backend/booster/bk_dist/booster/pkg/booster.go b/src/backend/booster/bk_dist/booster/pkg/booster.go index d97fb515..d6492e98 100644 --- a/src/backend/booster/bk_dist/booster/pkg/booster.go +++ b/src/backend/booster/bk_dist/booster/pkg/booster.go @@ -1326,7 +1326,7 @@ func (b *Booster) setToolChainWithJSON(tools *dcSDK.Toolchain) error { } func (b *Booster) checkPump() { - if b.config.Works.Pump { + if b.config.Works.Pump || b.config.Works.PumpCache { pumpdir := b.config.Works.PumpCacheDir if pumpdir == "" { pumpdir = dcUtil.GetPumpCacheDir() @@ -1334,11 +1334,10 @@ func (b *Booster) checkPump() { blog.Infof("booster: not found pump cache dir, do nothing") return } - - // fresh env of cache dir - os.Setenv(env.GetEnvKey(env.KeyExecutorPumpCacheDir), pumpdir) - b.config.Works.PumpCacheDir = pumpdir } + // fresh env of cache dir + os.Setenv(env.GetEnvKey(env.KeyExecutorPumpCacheDir), pumpdir) + b.config.Works.PumpCacheDir = pumpdir b.checkPumpCache(pumpdir) diff --git a/src/backend/booster/bk_dist/common/sdk/worker.go b/src/backend/booster/bk_dist/common/sdk/worker.go index 5c626757..c2c3cf28 100644 --- a/src/backend/booster/bk_dist/common/sdk/worker.go +++ b/src/backend/booster/bk_dist/common/sdk/worker.go @@ -64,6 +64,12 @@ const ( MinFileDescPriority FileDescPriority = 100 MaxFileDescPriority FileDescPriority = 0 + // pump cache模式，文件发送需要保证顺序，定义下面几个优先级(由高到底) + RealDirPriority FileDescPriority = 0 + LinkDirPriority FileDescPriority = 1 + RealFilePriority FileDescPriority = 2 + LinkFilePriority FileDescPriority = 3 + // 优先级先只定3个 PriorityLow = 0 PriorityMiddle = 1 diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go index 0c2b62d5..69dc417b 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go @@ -588,12 +588,12 @@ func (m *Mgr) SendStats(brief bool) error { _ = codec.EncJSON(message, &data) if _, _, err := m.request("POST", m.serverHost, messageURI, data); err != nil { - blog.Errorf("resource: send stats(detail %v) to server for task(%s) work(%s) failed: %v", + blog.Errorf("resource: send stats(brief %v) to server for task(%s) work(%s) failed: %v", brief, taskID, workID, err) return err } - blog.Infof("resource: success to send stats detail %v to server for task(%s) work(%s)", + blog.Infof("resource: success to send stats brief %v to server for task(%s) work(%s)", brief, taskID, workID) return nil } diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go index 645c1e8e..fc62c6ef 100644 --- a/src/backend/booster/bk_dist/handler/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/cc/handler.go @@ -227,19 +227,6 @@ func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) { }, nil } -func uniqArr(arr []string) []string { - newarr := make([]string, 0) - tempMap := make(map[string]bool, len(newarr)) - for _, v := range arr { - if tempMap[v] == false { - tempMap[v] = true - newarr = append(newarr, v) - } - } - - return newarr -} - func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) { data, err := os.ReadFile(dependf) if err != nil { @@ -248,7 +235,7 @@ func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) { sep := "\n" lines := strings.Split(string(data), sep) - uniqlines := uniqArr(lines) + uniqlines := commonUtil.UniqArr(lines) blog.Infof("cc: got %d uniq include file from file: %s", len(uniqlines), dependf) return commonUtil.GetFileInfo(uniqlines, false, false, dcPump.SupportPumpLstatByDir(cc.sandbox.Env)) @@ -311,7 +298,7 @@ func (cc *TaskCC) resolveDependFile(sep, workdir string, includes *[]string) err *includes = append(*includes, commonUtil.FormatFilePath(targetf)) // 如果是链接，则将相关指向的文件都包含进来 - fs := commonUtil.GetAllLinkFiles(targetf, workdir) + fs := commonUtil.GetAllLinkFiles(targetf) if len(fs) > 0 { *includes = append(*includes, fs...) } @@ -372,7 +359,7 @@ func (cc *TaskCC) copyPumpHeadFile(workdir string) error { return ErrorInvalidDependFile } - uniqlines := uniqArr(includes) + uniqlines := commonUtil.UniqArr(includes) // append symlink or symlinked if need links, _ := getIncludeLinks(cc.sandbox.Env, uniqlines) @@ -380,6 +367,12 @@ func (cc *TaskCC) copyPumpHeadFile(workdir string) error { uniqlines = append(uniqlines, links...) } + // TODO :将链接路径找出并放到前面 + linkdirs := commonUtil.GetAllLinkDir(uniqlines) + if len(linkdirs) > 0 { + uniqlines = append(linkdirs, uniqlines...) + } + // save to cc.pumpHeadFile newdata := strings.Join(uniqlines, sep) err = os.WriteFile(cc.pumpHeadFile, []byte(newdata), os.ModePerm) @@ -674,7 +667,7 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro Targetrelativepath: filepath.Dir(fpath), LinkTarget: f.LinkTarget, NoDuplicated: true, - // Priority: priority, + Priority: commonUtil.GetPriority(f), }) // priority++ // blog.Infof("cc: added include file:%s with modify time %d", fpath, modifyTime) diff --git a/src/backend/booster/bk_dist/handler/common/util.go b/src/backend/booster/bk_dist/handler/common/util.go index ef6d16f2..e3f562b1 100644 --- a/src/backend/booster/bk_dist/handler/common/util.go +++ b/src/backend/booster/bk_dist/handler/common/util.go @@ -21,6 +21,7 @@ import ( "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env" dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol" + dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" @@ -361,7 +362,7 @@ func FormatFilePath(f string) string { return f } -func GetAllLinkFiles(f, workdir string) []string { +func GetAllLinkFiles(f string) []string { fs := []string{} tempf := f // avoid dead loop @@ -374,7 +375,7 @@ func GetAllLinkFiles(f, workdir string) []string { originFile, err := os.Readlink(tempf) if err == nil { if !filepath.IsAbs(originFile) { - originFile, _ = filepath.Abs(filepath.Join(workdir, originFile)) + originFile, _ = filepath.Abs(filepath.Join(filepath.Dir(tempf), originFile)) } fs = append(fs, FormatFilePath(originFile)) @@ -397,3 +398,117 @@ func GetAllLinkFiles(f, workdir string) []string { return fs } + +func UniqArr(arr []string) []string { + newarr := make([]string, 0) + tempMap := make(map[string]bool, len(newarr)) + for _, v := range arr { + if tempMap[v] == false { + tempMap[v] = true + newarr = append(newarr, v) + } + } + + return newarr +} + +func getSubdirs(path string) []string { + var subdirs []string + + // 循环获取每级子目录 + for { + subdirs = append([]string{path}, subdirs...) + parent := filepath.Dir(path) + if parent == path { + break + } + path = parent + } + + return subdirs +} + +// 获取依赖文件的路径中是链接的路径 +func GetAllLinkDir(files []string) []string { + dirs := make([]string, 0, len(files)) + for _, f := range files { + dirs = append(dirs, filepath.Dir(f)) + } + + uniqdirs := UniqArr(dirs) + if len(uniqdirs) > 0 { + subdirs := []string{} + for _, d := range uniqdirs { + subdirs = append(subdirs, getSubdirs(d)...) + } + + uniqsubdirs := UniqArr(subdirs) + blog.Infof("common util: got all uniq sub dirs:%v", uniqsubdirs) + + linkdirs := []string{} + for _, d := range uniqsubdirs { + i := dcFile.Lstat(d) + if i.Basic().Mode()&os.ModeSymlink != 0 { + fs := GetAllLinkFiles(d) + if len(fs) > 0 { + for i := len(fs) - 1; i >= 0; i-- { + linkdirs = append(linkdirs, fs[i]) + } + linkdirs = append(linkdirs, d) + } + } + } + + blog.Infof("common util: got all link sub dirs:%v", linkdirs) + return linkdirs + } + + return nil +} + +func GetPriority(i *dcFile.Info) dcSDK.FileDescPriority { + isLink := i.Basic().Mode()&os.ModeSymlink != 0 + if !isLink { + if i.Basic().IsDir() { + return dcSDK.RealDirPriority + } else { + return dcSDK.RealFilePriority + } + } + + // symlink 需要判断是指向文件还是目录 + if i.LinkTarget != "" { + targetfs, err := GetFileInfo([]string{i.LinkTarget}, true, false, false) + if err == nil && len(targetfs) > 0 { + if targetfs[0].Basic().IsDir() { + return dcSDK.LinkDirPriority + } else { + return dcSDK.LinkFilePriority + } + } + } + + // 尝试读文件 + linkpath := i.Path() + targetPath, err := os.Readlink(linkpath) + if err != nil { + blog.Infof("common util: Error reading symbolic link: %v", err) + return dcSDK.LinkFilePriority + } + + // 获取符号链接指向路径的文件信息 + targetInfo, err := os.Stat(targetPath) + if err != nil { + blog.Infof("common util: Error getting target file info: %v", err) + return dcSDK.LinkFilePriority + } + + // 判断符号链接指向的路径是否是目录 + if targetInfo.IsDir() { + blog.Infof("common util: %s is a symbolic link to a directory", linkpath) + return dcSDK.LinkDirPriority + } else { + blog.Infof("common util: %s is a symbolic link, but not to a directory", linkpath) + return dcSDK.LinkFilePriority + } +} diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go index ab8340a2..5a5441d8 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go @@ -247,18 +247,18 @@ func (cc *TaskCC) getIncludeExe() (string, error) { return includePath, nil } -func uniqArr(arr []string) []string { - newarr := make([]string, 0) - tempMap := make(map[string]bool, len(newarr)) - for _, v := range arr { - if tempMap[v] == false { - tempMap[v] = true - newarr = append(newarr, v) - } - } +// func uniqArr(arr []string) []string { +// newarr := make([]string, 0) +// tempMap := make(map[string]bool, len(newarr)) +// for _, v := range arr { +// if tempMap[v] == false { +// tempMap[v] = true +// newarr = append(newarr, v) +// } +// } - return newarr -} +// return newarr +// } func (cc *TaskCC) analyzeIncludes(dependf string, workdir string) ([]*dcFile.Info, error) { data, err := ioutil.ReadFile(dependf) @@ -271,7 +271,7 @@ func (cc *TaskCC) analyzeIncludes(dependf string, workdir string) ([]*dcFile.Inf sep = "\r\n" } lines := strings.Split(string(data), sep) - uniqlines := uniqArr(lines) + uniqlines := commonUtil.UniqArr(lines) blog.Infof("cc: got %d uniq include file from file: %s", len(uniqlines), dependf) return commonUtil.GetFileInfo(uniqlines, false, false, dcPump.SupportPumpLstatByDir(cc.sandbox.Env)) @@ -360,7 +360,7 @@ func (cc *TaskCC) resolveDependFile(sep, workdir string, includes *[]string) err *includes = append(*includes, commonUtil.FormatFilePath(targetf)) // 如果是链接，则将相关指向的文件都包含进来 - fs := commonUtil.GetAllLinkFiles(targetf, workdir) + fs := commonUtil.GetAllLinkFiles(targetf) if len(fs) > 0 { *includes = append(*includes, fs...) } @@ -474,7 +474,7 @@ func (cc *TaskCC) copyPumpHeadFile(workdir string) error { // for i := range includes { // includes[i] = strings.Replace(includes[i], "\\", "/", -1) // } - uniqlines := uniqArr(includes) + uniqlines := commonUtil.UniqArr(includes) // TODO : append symlink or symlinked if need links, _ := getIncludeLinks(cc.sandbox.Env, uniqlines) @@ -482,6 +482,12 @@ func (cc *TaskCC) copyPumpHeadFile(workdir string) error { uniqlines = append(uniqlines, links...) } + // TODO :将链接路径找出并放到前面 + linkdirs := commonUtil.GetAllLinkDir(uniqlines) + if len(linkdirs) > 0 { + uniqlines = append(linkdirs, uniqlines...) + } + // TODO : save to cc.pumpHeadFile newdata := strings.Join(uniqlines, sep) err = ioutil.WriteFile(cc.pumpHeadFile, []byte(newdata), os.ModePerm) @@ -799,7 +805,7 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error) Targetrelativepath: filepath.Dir(fpath), LinkTarget: f.LinkTarget, NoDuplicated: true, - // Priority: priority, + Priority: commonUtil.GetPriority(f), }) // priority++ // blog.Infof("cc: added include file:%s with modify time %d", fpath, modifyTime) diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go index fa400401..84e003b9 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go @@ -360,18 +360,18 @@ func (cl *TaskCL) getIncludeExe() (string, error) { return includePath, nil } -func uniqArr(arr []string) []string { - newarr := make([]string, 0) - tempMap := make(map[string]bool, len(newarr)) - for _, v := range arr { - if tempMap[v] == false { - tempMap[v] = true - newarr = append(newarr, v) - } - } +// func uniqArr(arr []string) []string { +// newarr := make([]string, 0) +// tempMap := make(map[string]bool, len(newarr)) +// for _, v := range arr { +// if tempMap[v] == false { +// tempMap[v] = true +// newarr = append(newarr, v) +// } +// } - return newarr -} +// return newarr +// } func (cl *TaskCL) analyzeIncludes(f string, workdir string) ([]*dcFile.Info, error) { data, err := ioutil.ReadFile(f) @@ -380,7 +380,7 @@ func (cl *TaskCL) analyzeIncludes(f string, workdir string) ([]*dcFile.Info, err } lines := strings.Split(string(data), "\r\n") - uniqlines := uniqArr(lines) + uniqlines := commonUtil.UniqArr(lines) blog.Infof("cl: got %d uniq include file from file: %s", len(uniqlines), f) // if dcPump.SupportPumpStatCache(cl.sandbox.Env) { @@ -489,7 +489,7 @@ func (cl *TaskCL) copyPumpHeadFile(workdir string) error { includes = append(includes, commonUtil.FormatFilePath(l)) // 如果是链接，则将相关指向的文件都包含进来 - fs := commonUtil.GetAllLinkFiles(l, workdir) + fs := commonUtil.GetAllLinkFiles(l) if len(fs) > 0 { includes = append(includes, fs...) } @@ -508,7 +508,7 @@ func (cl *TaskCL) copyPumpHeadFile(workdir string) error { includes = append(includes, commonUtil.FormatFilePath(l)) // 如果是链接，则将相关指向的文件都包含进来 - fs := commonUtil.GetAllLinkFiles(l, workdir) + fs := commonUtil.GetAllLinkFiles(l) if len(fs) > 0 { includes = append(includes, fs...) } @@ -556,7 +556,7 @@ func (cl *TaskCL) copyPumpHeadFile(workdir string) error { // for i := range includes { // includes[i] = strings.Replace(includes[i], "/", "\\", -1) // } - uniqlines := uniqArr(includes) + uniqlines := commonUtil.UniqArr(includes) if dcPump.PumpCorrectCap(cl.sandbox.Env) { uniqlines, _ = commonUtil.CorrectPathCap(uniqlines) @@ -744,7 +744,7 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error) Filemode: fileMode, Targetrelativepath: filepath.Dir(fpath), NoDuplicated: true, - // Priority: priority, + Priority: commonUtil.GetPriority(f), }) // priority++