Merge branch 'master' of https://github.com/TencentBlueKing/bk-turbo …

…into dev_yanafu

src/backend/booster/bk_dist/booster/pkg/booster.go

@@ -1326,19 +1326,18 @@ func (b *Booster) setToolChainWithJSON(tools *dcSDK.Toolchain) error {
 }
 
 func (b *Booster) checkPump() {
-	if b.config.Works.Pump {
+	if b.config.Works.Pump || b.config.Works.PumpCache {
 		pumpdir := b.config.Works.PumpCacheDir
 		if pumpdir == "" {
 			pumpdir = dcUtil.GetPumpCacheDir()
 			if pumpdir == "" {
 				blog.Infof("booster: not found pump cache dir, do nothing")
 				return
 			}
-
-			// fresh env of cache dir
-			os.Setenv(env.GetEnvKey(env.KeyExecutorPumpCacheDir), pumpdir)
-			b.config.Works.PumpCacheDir = pumpdir
 		}
+		// fresh env of cache dir
+		os.Setenv(env.GetEnvKey(env.KeyExecutorPumpCacheDir), pumpdir)
+		b.config.Works.PumpCacheDir = pumpdir
 
 		b.checkPumpCache(pumpdir)
 

src/backend/booster/bk_dist/common/sdk/worker.go

@@ -64,6 +64,12 @@ const (
 	MinFileDescPriority FileDescPriority = 100
 	MaxFileDescPriority FileDescPriority = 0
 
+	// pump cache模式，文件发送需要保证顺序，定义下面几个优先级(由高到底)
+	RealDirPriority  FileDescPriority = 0
+	LinkDirPriority  FileDescPriority = 1
+	RealFilePriority FileDescPriority = 2
+	LinkFilePriority FileDescPriority = 3
+
 	// 优先级先只定3个
 	PriorityLow     = 0
 	PriorityMiddle  = 1

src/backend/booster/bk_dist/controller/pkg/manager/resource/mgr.go

@@ -588,12 +588,12 @@ func (m *Mgr) SendStats(brief bool) error {
 	_ = codec.EncJSON(message, &data)
 
 	if _, _, err := m.request("POST", m.serverHost, messageURI, data); err != nil {
-		blog.Errorf("resource: send stats(detail %v) to server for task(%s) work(%s) failed: %v",
+		blog.Errorf("resource: send stats(brief %v) to server for task(%s) work(%s) failed: %v",
 			brief, taskID, workID, err)
 		return err
 	}
 
-	blog.Infof("resource: success to send stats detail %v to server for task(%s) work(%s)",
+	blog.Infof("resource: success to send stats brief %v to server for task(%s) work(%s)",
 		brief, taskID, workID)
 	return nil
 }

src/backend/booster/bk_dist/handler/cc/handler.go

@@ -227,19 +227,6 @@ func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
 	}, nil
 }
 
-func uniqArr(arr []string) []string {
-	newarr := make([]string, 0)
-	tempMap := make(map[string]bool, len(newarr))
-	for _, v := range arr {
-		if tempMap[v] == false {
-			tempMap[v] = true
-			newarr = append(newarr, v)
-		}
-	}
-
-	return newarr
-}
-
 func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) {
 	data, err := os.ReadFile(dependf)
 	if err != nil {
@@ -248,7 +235,7 @@ func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) {
 
 	sep := "\n"
 	lines := strings.Split(string(data), sep)
-	uniqlines := uniqArr(lines)
+	uniqlines := commonUtil.UniqArr(lines)
 	blog.Infof("cc: got %d uniq include file from file: %s", len(uniqlines), dependf)
 
 	return commonUtil.GetFileInfo(uniqlines, false, false, dcPump.SupportPumpLstatByDir(cc.sandbox.Env))
@@ -311,7 +298,7 @@ func (cc *TaskCC) resolveDependFile(sep, workdir string, includes *[]string) err
 				*includes = append(*includes, commonUtil.FormatFilePath(targetf))
 
 				// 如果是链接，则将相关指向的文件都包含进来
-				fs := commonUtil.GetAllLinkFiles(targetf, workdir)
+				fs := commonUtil.GetAllLinkFiles(targetf)
 				if len(fs) > 0 {
 					*includes = append(*includes, fs...)
 				}
@@ -372,14 +359,20 @@ func (cc *TaskCC) copyPumpHeadFile(workdir string) error {
 		return ErrorInvalidDependFile
 	}
 
-	uniqlines := uniqArr(includes)
+	uniqlines := commonUtil.UniqArr(includes)
 
 	// append symlink or symlinked if need
 	links, _ := getIncludeLinks(cc.sandbox.Env, uniqlines)
 	if links != nil {
 		uniqlines = append(uniqlines, links...)
 	}
 
+	// TODO :将链接路径找出并放到前面
+	linkdirs := commonUtil.GetAllLinkDir(uniqlines)
+	if len(linkdirs) > 0 {
+		uniqlines = append(linkdirs, uniqlines...)
+	}
+
 	// save to cc.pumpHeadFile
 	newdata := strings.Join(uniqlines, sep)
 	err = os.WriteFile(cc.pumpHeadFile, []byte(newdata), os.ModePerm)
@@ -674,7 +667,7 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro
 				Targetrelativepath: filepath.Dir(fpath),
 				LinkTarget:         f.LinkTarget,
 				NoDuplicated:       true,
-				// Priority:           priority,
+				Priority:           commonUtil.GetPriority(f),
 			})
 			// priority++
 			// blog.Infof("cc: added include file:%s with modify time %d", fpath, modifyTime)

src/backend/booster/bk_dist/handler/common/util.go

@@ -21,6 +21,7 @@ import (
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/env"
 	dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol"
+	dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk"
 	dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog"
@@ -361,7 +362,7 @@ func FormatFilePath(f string) string {
 	return f
 }
 
-func GetAllLinkFiles(f, workdir string) []string {
+func GetAllLinkFiles(f string) []string {
 	fs := []string{}
 	tempf := f
 	// avoid dead loop
@@ -374,7 +375,7 @@ func GetAllLinkFiles(f, workdir string) []string {
 			originFile, err := os.Readlink(tempf)
 			if err == nil {
 				if !filepath.IsAbs(originFile) {
-					originFile, _ = filepath.Abs(filepath.Join(workdir, originFile))
+					originFile, _ = filepath.Abs(filepath.Join(filepath.Dir(tempf), originFile))
 				}
 				fs = append(fs, FormatFilePath(originFile))
 
@@ -397,3 +398,117 @@ func GetAllLinkFiles(f, workdir string) []string {
 
 	return fs
 }
+
+func UniqArr(arr []string) []string {
+	newarr := make([]string, 0)
+	tempMap := make(map[string]bool, len(newarr))
+	for _, v := range arr {
+		if tempMap[v] == false {
+			tempMap[v] = true
+			newarr = append(newarr, v)
+		}
+	}
+
+	return newarr
+}
+
+func getSubdirs(path string) []string {
+	var subdirs []string
+
+	// 循环获取每级子目录
+	for {
+		subdirs = append([]string{path}, subdirs...)
+		parent := filepath.Dir(path)
+		if parent == path {
+			break
+		}
+		path = parent
+	}
+
+	return subdirs
+}
+
+// 获取依赖文件的路径中是链接的路径
+func GetAllLinkDir(files []string) []string {
+	dirs := make([]string, 0, len(files))
+	for _, f := range files {
+		dirs = append(dirs, filepath.Dir(f))
+	}
+
+	uniqdirs := UniqArr(dirs)
+	if len(uniqdirs) > 0 {
+		subdirs := []string{}
+		for _, d := range uniqdirs {
+			subdirs = append(subdirs, getSubdirs(d)...)
+		}
+
+		uniqsubdirs := UniqArr(subdirs)
+		blog.Infof("common util: got all uniq sub dirs:%v", uniqsubdirs)
+
+		linkdirs := []string{}
+		for _, d := range uniqsubdirs {
+			i := dcFile.Lstat(d)
+			if i.Basic().Mode()&os.ModeSymlink != 0 {
+				fs := GetAllLinkFiles(d)
+				if len(fs) > 0 {
+					for i := len(fs) - 1; i >= 0; i-- {
+						linkdirs = append(linkdirs, fs[i])
+					}
+					linkdirs = append(linkdirs, d)
+				}
+			}
+		}
+
+		blog.Infof("common util: got all link sub dirs:%v", linkdirs)
+		return linkdirs
+	}
+
+	return nil
+}
+
+func GetPriority(i *dcFile.Info) dcSDK.FileDescPriority {
+	isLink := i.Basic().Mode()&os.ModeSymlink != 0
+	if !isLink {
+		if i.Basic().IsDir() {
+			return dcSDK.RealDirPriority
+		} else {
+			return dcSDK.RealFilePriority
+		}
+	}
+
+	// symlink 需要判断是指向文件还是目录
+	if i.LinkTarget != "" {
+		targetfs, err := GetFileInfo([]string{i.LinkTarget}, true, false, false)
+		if err == nil && len(targetfs) > 0 {
+			if targetfs[0].Basic().IsDir() {
+				return dcSDK.LinkDirPriority
+			} else {
+				return dcSDK.LinkFilePriority
+			}
+		}
+	}
+
+	// 尝试读文件
+	linkpath := i.Path()
+	targetPath, err := os.Readlink(linkpath)
+	if err != nil {
+		blog.Infof("common util: Error reading symbolic link: %v", err)
+		return dcSDK.LinkFilePriority
+	}
+
+	// 获取符号链接指向路径的文件信息
+	targetInfo, err := os.Stat(targetPath)
+	if err != nil {
+		blog.Infof("common util: Error getting target file info: %v", err)
+		return dcSDK.LinkFilePriority
+	}
+
+	// 判断符号链接指向的路径是否是目录
+	if targetInfo.IsDir() {
+		blog.Infof("common util: %s is a symbolic link to a directory", linkpath)
+		return dcSDK.LinkDirPriority
+	} else {
+		blog.Infof("common util: %s is a symbolic link, but not to a directory", linkpath)
+		return dcSDK.LinkFilePriority
+	}
+}

src/backend/booster/bk_dist/handler/ue4/cc/handler.go

@@ -247,18 +247,18 @@ func (cc *TaskCC) getIncludeExe() (string, error) {
 	return includePath, nil
 }
 
-func uniqArr(arr []string) []string {
-	newarr := make([]string, 0)
-	tempMap := make(map[string]bool, len(newarr))
-	for _, v := range arr {
-		if tempMap[v] == false {
-			tempMap[v] = true
-			newarr = append(newarr, v)
-		}
-	}
+// func uniqArr(arr []string) []string {
+// 	newarr := make([]string, 0)
+// 	tempMap := make(map[string]bool, len(newarr))
+// 	for _, v := range arr {
+// 		if tempMap[v] == false {
+// 			tempMap[v] = true
+// 			newarr = append(newarr, v)
+// 		}
+// 	}
 
-	return newarr
-}
+// 	return newarr
+// }
 
 func (cc *TaskCC) analyzeIncludes(dependf string, workdir string) ([]*dcFile.Info, error) {
 	data, err := ioutil.ReadFile(dependf)
@@ -271,7 +271,7 @@ func (cc *TaskCC) analyzeIncludes(dependf string, workdir string) ([]*dcFile.Inf
 		sep = "\r\n"
 	}
 	lines := strings.Split(string(data), sep)
-	uniqlines := uniqArr(lines)
+	uniqlines := commonUtil.UniqArr(lines)
 	blog.Infof("cc: got %d uniq include file from file: %s", len(uniqlines), dependf)
 
 	return commonUtil.GetFileInfo(uniqlines, false, false, dcPump.SupportPumpLstatByDir(cc.sandbox.Env))
@@ -360,7 +360,7 @@ func (cc *TaskCC) resolveDependFile(sep, workdir string, includes *[]string) err
 				*includes = append(*includes, commonUtil.FormatFilePath(targetf))
 
 				// 如果是链接，则将相关指向的文件都包含进来
-				fs := commonUtil.GetAllLinkFiles(targetf, workdir)
+				fs := commonUtil.GetAllLinkFiles(targetf)
 				if len(fs) > 0 {
 					*includes = append(*includes, fs...)
 				}
@@ -474,14 +474,20 @@ func (cc *TaskCC) copyPumpHeadFile(workdir string) error {
 	// for i := range includes {
 	// 	includes[i] = strings.Replace(includes[i], "\\", "/", -1)
 	// }
-	uniqlines := uniqArr(includes)
+	uniqlines := commonUtil.UniqArr(includes)
 
 	// TODO : append symlink or symlinked if need
 	links, _ := getIncludeLinks(cc.sandbox.Env, uniqlines)
 	if links != nil {
 		uniqlines = append(uniqlines, links...)
 	}
 
+	// TODO :将链接路径找出并放到前面
+	linkdirs := commonUtil.GetAllLinkDir(uniqlines)
+	if len(linkdirs) > 0 {
+		uniqlines = append(linkdirs, uniqlines...)
+	}
+
 	// TODO : save to cc.pumpHeadFile
 	newdata := strings.Join(uniqlines, sep)
 	err = ioutil.WriteFile(cc.pumpHeadFile, []byte(newdata), os.ModePerm)
@@ -799,7 +805,7 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
 				Targetrelativepath: filepath.Dir(fpath),
 				LinkTarget:         f.LinkTarget,
 				NoDuplicated:       true,
-				// Priority:           priority,
+				Priority:           commonUtil.GetPriority(f),
 			})
 			// priority++
 			// blog.Infof("cc: added include file:%s with modify time %d", fpath, modifyTime)