From f783946810f3d95cae96289c82ca84980e5cf36c Mon Sep 17 00:00:00 2001 From: tbs60 Date: Mon, 26 Aug 2024 17:39:10 +0800 Subject: [PATCH 1/2] fix: add search link dirs, issue: #285 --- .../booster/bk_dist/booster/pkg/booster.go | 71 +++++++++++++------ .../booster/bk_dist/booster/pkg/util.go | 5 ++ 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/backend/booster/bk_dist/booster/pkg/booster.go b/src/backend/booster/bk_dist/booster/pkg/booster.go index ef2d1163..d97fb515 100644 --- a/src/backend/booster/bk_dist/booster/pkg/booster.go +++ b/src/backend/booster/bk_dist/booster/pkg/booster.go @@ -1343,15 +1343,15 @@ func (b *Booster) checkPump() { b.checkPumpCache(pumpdir) if b.config.Works.PumpSearchLink && runtime.GOOS == "darwin" { + dirs := []string{} // 获取默认xcode的路径 xcodepath, err := getXcodeIncludeLinkDir() if err != nil || xcodepath == nil { blog.Infof("booster: get default xcode path with error:%v", err) - return + } else { + // 得到所有需要搜索的目录（包括默认xcode和用户指定的） + dirs = xcodepath } - - // 得到所有需要搜索的目录（包括默认xcode和用户指定的） - dirs := xcodepath dirs = append(dirs, b.config.Works.PumpSearchLinkDir...) // 搜索所有symlink @@ -1405,27 +1405,50 @@ func getXcodeIncludeLinkDir() ([]string, error) { return nil, err } - xcodepath := filepath.Join(strings.Trim(string(stdout), "\r\n "), "Platforms/MacOSX.platform/Developer/SDKs") - info, err := os.Stat(xcodepath) - if info != nil && (err == nil || os.IsExist(err)) { - fis, err := ioutil.ReadDir(xcodepath) - if err != nil { - return nil, err + // get all dirs in Platforms + platformspath := filepath.Join(strings.Trim(string(stdout), "\r\n "), "Platforms") + var subdirs []string + + entries, err := os.ReadDir(platformspath) + if err != nil { + return nil, err + } + + for _, entry := range entries { + if entry.IsDir() { + absPath, err := filepath.Abs(filepath.Join(platformspath, entry.Name())) + if err != nil { + return nil, err + } + subdirs = append(subdirs, absPath) } + } + + // get all target dirs + resultdirs := []string{} + for _, v := range subdirs { + // xcodepath := filepath.Join(strings.Trim(string(stdout), "\r\n "), "Platforms/MacOSX.platform/Developer/SDKs") + xcodepath := filepath.Join(v, "Developer/SDKs") + info, err := os.Stat(xcodepath) + if info != nil && (err == nil || os.IsExist(err)) { + fis, err := ioutil.ReadDir(xcodepath) + if err != nil { + return nil, err + } - for _, fi := range fis { - if fi.Mode()&os.ModeSymlink != 0 { - linkfile := filepath.Join(xcodepath, fi.Name()) - originFile, err := os.Readlink(linkfile) + for _, fi := range fis { + if fi.Mode()&os.ModeSymlink != 0 { + linkfile := filepath.Join(xcodepath, fi.Name()) + originFile, err := os.Readlink(linkfile) - if err != nil { - blog.Warnf("booster: readlink %s with error:%v", linkfile, err) - continue - } + if err != nil { + blog.Warnf("booster: readlink %s with error:%v", linkfile, err) + continue + } - blog.Infof("booster: Resolved symlink %s to %s", linkfile, originFile) + blog.Infof("booster: Resolved symlink %s to %s", linkfile, originFile) - if strings.HasSuffix(originFile, "MacOSX.sdk") { + // if strings.HasSuffix(originFile, "MacOSX.sdk") { blog.Infof("booster: found target link dir %s", linkfile) xcodepath1 := filepath.Join(linkfile, "usr/include") @@ -1435,13 +1458,17 @@ func getXcodeIncludeLinkDir() ([]string, error) { } xcodepath2 := filepath.Join(originFile, "usr/include") - return []string{xcodepath1, xcodepath2}, nil + resultdirs = append(resultdirs, xcodepath1) + resultdirs = append(resultdirs, xcodepath2) + + // return []string{xcodepath1, xcodepath2}, nil + // } } } } } - return nil, nil + return resultdirs, nil } func getLinkFile(pumpdir string, xcodepath string) string { diff --git a/src/backend/booster/bk_dist/booster/pkg/util.go b/src/backend/booster/bk_dist/booster/pkg/util.go index e85446b9..675e66ec 100644 --- a/src/backend/booster/bk_dist/booster/pkg/util.go +++ b/src/backend/booster/bk_dist/booster/pkg/util.go @@ -248,6 +248,11 @@ func searchSymlink(dirPth string, files map[string]string) (err error) { } err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error { + if err != nil { + blog.Infof("booster: search path:%s failed with error:%v", dirPth, err) + return err + } + if fi.IsDir() { return nil } From 985355a786378b3546c2f9e97961e35d0faa7b6d Mon Sep 17 00:00:00 2001 From: tbs60 Date: Tue, 27 Aug 2024 11:53:34 +0800 Subject: [PATCH 2/2] fix: adjust file info from cache, issue: #287 --- .../booster/bk_dist/handler/common/util.go | 10 ++++++-- .../pkg/client/bkcommondist_protocol.go | 24 +++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/backend/booster/bk_dist/handler/common/util.go b/src/backend/booster/bk_dist/handler/common/util.go index e18a5e16..ef6d16f2 100644 --- a/src/backend/booster/bk_dist/handler/common/util.go +++ b/src/backend/booster/bk_dist/handler/common/util.go @@ -116,6 +116,8 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo tempis := make(map[string]*dcFile.Info, len(notfound)) for _, notf := range notfound { tempf := notf + try := 0 + maxtry := 10 for { var i *dcFile.Info if statbysearchdir { @@ -124,6 +126,7 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo i = dcFile.Lstat(tempf) } tempis[tempf] = i + try++ if !i.Exist() { if mustexisted { @@ -156,8 +159,11 @@ func GetFileInfo(fs []string, mustexisted bool, notdir bool, statbysearchdir boo } // 如果是链接，并且指向了其它文件，则需要将指向的文件也包含进来 - loopagain = true - tempf = originFile + // 增加寻找次数限制，避免死循环 + if try < maxtry { + loopagain = true + tempf = originFile + } } else { blog.Infof("common util: symlink %s Readlink error:%s", tempf, err) } diff --git a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go index 06566f4b..d6104e1b 100644 --- a/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go +++ b/src/backend/booster/bk_dist/worker/pkg/client/bkcommondist_protocol.go @@ -837,12 +837,28 @@ func encodeSendFileReq( } fullpath := sandbox.GetAbsPath(f.FilePath) - size := f.FileSize - md5 := f.Md5 + + // TODO : fresh file info here, avoid file info changed + newlyInfo := dcFile.Lstat(fullpath) + if !newlyInfo.Exist() { + blog.Warnf("file %f not existed when encode send request", fullpath) + continue + } + + size := newlyInfo.Size() + md5 := "" + if f.Md5 != "" { + md5, _ = newlyInfo.Md5() + } + filemode := newlyInfo.Mode32() + modifytime := newlyInfo.ModifyTime64() + + // size := f.FileSize + // md5 := f.Md5 targetrelativepath := f.Targetrelativepath - filemode := f.Filemode + // filemode := f.Filemode linkTarget := f.LinkTarget - modifytime := f.Lastmodifytime + // modifytime := f.Lastmodifytime if size <= 0 { pbbody.Inputfiles = append(pbbody.Inputfiles, &protocol.PBFileDesc{