Dev tming

src/backend/booster/bk_dist/common/file/file.go

@@ -82,16 +82,33 @@ func GetFileInfoByEnumDir(fp string) *Info {
 	}
 }
 
+type FileType int
+
+const (
+	Unknown FileType = 0
+
+	RealFile FileType = 10
+	RealDir  FileType = 11
+	LinkFile FileType = 12
+	LinkDir  FileType = 13
+)
+
 // Info describe the os.FileInfo and handle some actions
 type Info struct {
 	filePath   string
 	LinkTarget string
+	FileType   FileType
 
 	// info and err are return from os.Stat
 	info os.FileInfo
 	err  error
 }
 
+// Key return the uniq key of this file info
+func (i *Info) Key() string {
+	return i.filePath
+}
+
 // Path return the file path
 func (i *Info) Path() string {
 	return i.filePath
@@ -224,13 +241,15 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo
 		tempf := notf
 		try := 0
 		maxtry := 10
+		var oldi *Info
 		for {
 			var i *Info
 			if statbysearchdir {
 				i = GetFileInfoByEnumDir(tempf)
 			} else {
 				i = Lstat(tempf)
 			}
+			i.FileType = RealFile
 			tempis[tempf] = i
 			try++
 
@@ -279,6 +298,30 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo
 				continue
 			}
 
+			// 根据当前文件属性，给上一次的文件属性赋值
+			if loopagain { // 需要等链接的属性
+				i.FileType = LinkFile // 先假设是指向普通文件的链接
+				if oldi != nil {
+					oldi.FileType = LinkFile
+					blog.Infof("common util: set %s to LinkFile by assume", oldi.filePath)
+				}
+			} else {
+				if i.Basic().IsDir() {
+					i.FileType = RealDir
+					if oldi != nil {
+						oldi.FileType = LinkDir
+						blog.Infof("common util: set %s to LinkDir", oldi.filePath)
+					}
+				} else {
+					i.FileType = RealFile
+					if oldi != nil {
+						oldi.FileType = LinkFile
+						blog.Infof("common util: set %s to LinkFile", oldi.filePath)
+					}
+				}
+			}
+			oldi = i
+
 			is = append(is, i)
 
 			if !loopagain {
@@ -298,3 +341,21 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo
 
 	return is, nil
 }
+
+func Uniq(input []*Info) []*Info {
+	if input == nil {
+		return input
+	}
+
+	newarr := make([]*Info, 0, len(input)/2)
+	tempMap := make(map[string]struct{}, len(newarr))
+	for _, v := range input {
+		if _, ok := tempMap[v.Key()]; !ok {
+			tempMap[v.Key()] = struct{}{}
+			newarr = append(newarr, v)
+		}
+	}
+
+	return newarr
+
+}

src/backend/booster/bk_dist/common/sdk/worker.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"os"
 
+	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file"
 	dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file"
 	"github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol"
 	dcProtocol "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol"
@@ -186,6 +187,17 @@ type BKSlotRspAck struct {
 }
 
 func GetPriority(i *dcFile.Info) FileDescPriority {
+	switch i.FileType {
+	case file.RealDir:
+		return RealDirPriority
+	case file.LinkDir:
+		return LinkDirPriority
+	case file.RealFile:
+		return RealFilePriority
+	case file.LinkFile:
+		return LinkFilePriority
+	}
+
 	isLink := i.Basic().Mode()&os.ModeSymlink != 0
 	if !isLink {
 		if i.Basic().IsDir() {

src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go

@@ -665,7 +665,7 @@ func (m *Mgr) ensureFilesWithPriority(
 	fileDetails []*types.FilesDetails) ([]string, error) {
 
 	// 刷新优先级，windows的先不实现
-	if runtime.GOOS != "windows" {
+	if runtime.GOOS != "windows" && runtime.GOOS != "darwin" {
 		freshPriority(fileDetails)
 		for _, v := range fileDetails {
 			blog.Debugf("remote: after fresh Priority, file:%+v", *v)

src/backend/booster/bk_dist/controller/pkg/manager/remote/utils.go

@@ -204,44 +204,62 @@ func calculateDependencies(fileDetails []*types.FilesDetails) [][]int {
 	return dependencies
 }
 
+// 判断s2是否是s1的子串
 func isSubString(s1, s2 string) bool {
 	return len(s1) > len(s2) &&
 		strings.HasPrefix(s1, s2) &&
 		s2 != "/"
 }
 
 // dirDepend 检查 s1的目录 是否依赖 s2的目录
+// 优化规则：只有s2是指向目录的链接时，才需要判断依赖关系；
+// 因为普通目录（非链接）不影响远端路径的生成；而指向目录的链接，会在保存依赖时提炼出来
 func dirDepend(s1, s2 *types.FilesDetails) bool {
-	is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority
-	is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority
-	// 如果s1是文件，s2是目录
-	if is1File {
-		if is2File { // 如果s1是文件，s2是文件
-			if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) {
-				return true
-			}
-		} else { // 如果s1是文件，s2是目录
-			if isSubString(s1.File.FilePath, s2.File.FilePath) {
-				return true
-			}
-		}
-	} else {
-		if is2File { // 如果s1是目录，s2是文件
-			if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) {
-				return true
-			}
-		} else { // 如果s1是目录，s2是目录
-			if isSubString(s1.File.FilePath, s2.File.FilePath) {
-				return true
-			}
-		}
+	if s2.File.Priority != sdk.LinkDirPriority {
+		return false
+	}
+
+	if isSubString(s1.File.FilePath, s2.File.FilePath) {
+		return true
 	}
 
 	return false
+
+	// is1File := s1.File.Priority == sdk.RealFilePriority || s1.File.Priority == sdk.LinkFilePriority
+	// is2File := s2.File.Priority == sdk.RealFilePriority || s2.File.Priority == sdk.LinkFilePriority
+
+	// // 如果s1是文件，s2是目录
+	// if is1File {
+	// 	if is2File { // 如果s1是文件，s2是文件
+	// 		if isSubString(s1.File.Targetrelativepath, s2.File.Targetrelativepath) {
+	// 			return true
+	// 		}
+	// 	} else { // 如果s1是文件，s2是目录
+	// 		if isSubString(s1.File.FilePath, s2.File.FilePath) {
+	// 			return true
+	// 		}
+	// 	}
+	// } else {
+	// 	if is2File { // 如果s1是目录，s2是文件
+	// 		if isSubString(s1.File.FilePath, s2.File.Targetrelativepath) {
+	// 			return true
+	// 		}
+	// 	} else { // 如果s1是目录，s2是目录
+	// 		if isSubString(s1.File.FilePath, s2.File.FilePath) {
+	// 			return true
+	// 		}
+	// 	}
+	// }
+
+	// return false
 }
 
 // linkDepend 检查 s1 是否链接到了 s2
 func linkDepend(s1, s2 *types.FilesDetails) bool {
+	if s1.File.Priority != sdk.LinkDirPriority && s1.File.Priority != sdk.LinkFilePriority {
+		return false
+	}
+
 	if s1.File.LinkTarget != "" && s1.File.LinkTarget == s2.File.FilePath {
 		return true
 	}
@@ -334,9 +352,11 @@ func printLeftDepend(fileDetails []*types.FilesDetails, dependencies [][]int) {
 		if fileDetails[i].File.Priority < 0 {
 			dependfiles := []string{}
 			for _, v := range dependencies[i] {
-				dependPriority := int(fileDetails[v].File.Priority)
-				if dependPriority < 0 {
-					dependfiles = append(dependfiles, fileDetails[v].File.FilePath)
+				if v >= 0 {
+					dependPriority := int(fileDetails[v].File.Priority)
+					if dependPriority < 0 {
+						dependfiles = append(dependfiles, fileDetails[v].File.FilePath)
+					}
 				}
 			}
 			blog.Warnf("remote util: after max try, %s wait %s",

src/backend/booster/bk_dist/handler/cc/handler.go

@@ -223,14 +223,15 @@ func (cc *TaskCC) FinalExecute(args []string) {
 
 // GetFilterRules add file send filter
 func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
-	return []dcSDK.FilterRuleItem{
-		{
-			Rule:       dcSDK.FilterRuleFileSuffix,
-			Operator:   dcSDK.FilterRuleOperatorEqual,
-			Standard:   ".gch",
-			HandleType: dcSDK.FilterRuleHandleAllDistribution,
-		},
-	}, nil
+	// return []dcSDK.FilterRuleItem{
+	// 	{
+	// 		Rule:       dcSDK.FilterRuleFileSuffix,
+	// 		Operator:   dcSDK.FilterRuleOperatorEqual,
+	// 		Standard:   ".gch",
+	// 		HandleType: dcSDK.FilterRuleHandleAllDistribution,
+	// 	},
+	// }, nil
+	return nil, nil
 }
 
 func (cc *TaskCC) analyzeIncludes(dependf string) ([]*dcFile.Info, error) {
@@ -679,8 +680,6 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro
 	tstart = tend
 
 	if err == nil {
-		blog.Infof("cc: parse command,got total %d includes files", len(includes))
-
 		// add pch file as input
 		if pchfile != "" {
 			// includes = append(includes, pchfile)
@@ -699,6 +698,10 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro
 			}
 		}
 
+		oldlen := len(includes)
+		includes = dcFile.Uniq(includes)
+		blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes))
+
 		inputFiles := []dcSDK.FileDesc{}
 		// priority := dcSDK.MaxFileDescPriority
 		for _, f := range includes {

src/backend/booster/bk_dist/handler/ue4/cc/handler.go

@@ -225,14 +225,15 @@ func (cc *TaskCC) FinalExecute(args []string) {
 
 // GetFilterRules add file send filter
 func (cc *TaskCC) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
-	return []dcSDK.FilterRuleItem{
-		{
-			Rule:       dcSDK.FilterRuleFileSuffix,
-			Operator:   dcSDK.FilterRuleOperatorEqual,
-			Standard:   ".gch",
-			HandleType: dcSDK.FilterRuleHandleAllDistribution,
-		},
-	}, nil
+	// return []dcSDK.FilterRuleItem{
+	// 	{
+	// 		Rule:       dcSDK.FilterRuleFileSuffix,
+	// 		Operator:   dcSDK.FilterRuleOperatorEqual,
+	// 		Standard:   ".gch",
+	// 		HandleType: dcSDK.FilterRuleHandleAllDistribution,
+	// 	},
+	// }, nil
+	return nil, nil
 }
 
 func (cc *TaskCC) getIncludeExe() (string, error) {
@@ -816,8 +817,6 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
 	// }
 
 	if err == nil {
-		blog.Infof("cc: parse command,got total %d includes files", len(includes))
-
 		// add pch file as input
 		if pchfile != "" {
 			// includes = append(includes, pchfile)
@@ -836,6 +835,10 @@ func (cc *TaskCC) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
 			}
 		}
 
+		oldlen := len(includes)
+		includes = dcFile.Uniq(includes)
+		blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes))
+
 		inputFiles := []dcSDK.FileDesc{}
 		// priority := dcSDK.MaxFileDescPriority
 		for _, f := range includes {

src/backend/booster/bk_dist/handler/ue4/cl/handler.go

@@ -339,13 +339,14 @@ func (cl *TaskCL) FinalExecute(args []string) {
 
 // GetFilterRules add file send filter
 func (cl *TaskCL) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
-	return []dcSDK.FilterRuleItem{
-		{
-			Rule:     dcSDK.FilterRuleFileSuffix,
-			Operator: dcSDK.FilterRuleOperatorEqual,
-			Standard: ".pch",
-		},
-	}, nil
+	// return []dcSDK.FilterRuleItem{
+	// 	{
+	// 		Rule:     dcSDK.FilterRuleFileSuffix,
+	// 		Operator: dcSDK.FilterRuleOperatorEqual,
+	// 		Standard: ".pch",
+	// 	},
+	// }, nil
+	return nil, nil
 }
 
 func (cl *TaskCL) getIncludeExe() (string, error) {
@@ -719,8 +720,6 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
 	tstart = tend
 
 	if err == nil {
-		blog.Infof("cl: parse command,got total %d includes files", len(includes))
-
 		// add pch file as input
 		if pchfile != "" {
 			// includes = append(includes, pchfile)
@@ -739,6 +738,10 @@ func (cl *TaskCL) trypump(command []string) (*dcSDK.BKDistCommand, error, error)
 			}
 		}
 
+		oldlen := len(includes)
+		includes = dcFile.Uniq(includes)
+		blog.Infof("cc: parse command,got total %d uniq %d includes files", oldlen, len(includes))
+
 		inputFiles := []dcSDK.FileDesc{}
 		// priority := dcSDK.MaxFileDescPriority
 		for _, f := range includes {

src/backend/booster/bk_dist/handler/ue4/clfilter/handler.go

@@ -187,13 +187,14 @@ func (cf *TaskCLFilter) FinalExecute(args []string) {
 
 // GetFilterRules add file send filter
 func (cf *TaskCLFilter) GetFilterRules() ([]dcSDK.FilterRuleItem, error) {
-	return []dcSDK.FilterRuleItem{
-		{
-			Rule:     dcSDK.FilterRuleFileSuffix,
-			Operator: dcSDK.FilterRuleOperatorEqual,
-			Standard: ".pch",
-		},
-	}, nil
+	// return []dcSDK.FilterRuleItem{
+	// 	{
+	// 		Rule:     dcSDK.FilterRuleFileSuffix,
+	// 		Operator: dcSDK.FilterRuleOperatorEqual,
+	// 		Standard: ".pch",
+	// 	},
+	// }, nil
+	return nil, nil
 }
 
 func (cf *TaskCLFilter) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKDistCommonError) {